Found 298 jobs
Ignoring malparsed backtrace:  ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
,  1: /lib64/libpthread.so.0(+0x12cf0) [0x7f2181c7ecf0]
,  2: __pthread_mutex_lock()

Ignoring malparsed backtrace: 2023-03-25T02:01:50.041 INFO:teuthology.orchestra.run.smithi087.stdout:{"pg_ready":true,"pg_map":{"version":408,"stamp":"2023-03-25T02:01:48.868189+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":7100368,"num_objects":22,"num_object_clones":0,"num_object_copies":44,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":540,"num_write_kb":8438,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":19,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":500,"ondisk_log_size":500,"up":42,"acting":42,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":42,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":629145600,"kb_used":47288,"kb_used_data":15176,"kb_used_omap":0,"kb_used_meta":32064,"kb_avail":629098312,"statfs":{"total":644245094400,"available":644196671488,"internally_reserved":0,"allocated":15540224,"data_stored":14598440,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":32833536},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002186"},"pg_stats":[{"pgid":"3.b","version":"20'26","reported_seq":36,"reported_epoch":20,"state":"active+clean","last_fresh":"2023-03-25T01:49:07.722753+0000","last_change":"2023-03-25T01:48:56.990468+0000","last_active":"2023-03-25T01:49:07.722753+0000","last_peered":"2023-03-25T01:49:07.722753+0000","last_clean":"2023-03-25T01:49:07.722753+0000","last_became_active":"2023-03-25T01:48:56.990194+0000","last_became_peered":"2023-03-25T01:48:56.990194+0000","last_unstale":"2023-03-25T01:49:07.722753+0000","last_undegraded":"2023-03-25T01:49:07.722753+0000","last_fullsized":"2023-03-25T01:49:07.722753+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T01:48:55.982031+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T01:48:55.982031+0000","last_clean_scrub_stamp":"2023-03-25T01:48:55.982031+0000","objects_scrubbed":0,"log_size":26,"log_dups_size":0,"ondisk_log_size":26,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T07:51:55.253730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":330000,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":26,"num_write_kb":335,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":10,"reported_epoch":19,"state":"active+clean","last_fresh":"2023-03-25T01:48:58.910560+0000","last_change":"2023-03-25T01:48:56.990354+0000","last_active":"2023-03-25T01:48:58.910560+0000","last_peered":"2023-03-25T01:48:58.910560+0000","last_clean":"2023-03-25T01:48:58.910560+0000","last_became_active":"2023-03-25T01:48:56.989754+0000","last_became_peered":"2023-03-25T01:48:56.989754+0000","last_unstale":"2023-03-25T01:48:58.910560+0000","last_undegraded":"2023-03-25T01:48:58.910560+0000","last_fullsized":"2023-03-25T01:48:58.910560+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T01:48:55.982031+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T01:48:55.982031+0000","last_clean_scrub_stamp":"2023-03-25T01:48:55.982031+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:18:38.285450+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"20'64","reported_seq":74,"reported_epoch":20,"state":"active+clean","last_fresh":"2023-03-25T01:49:12.193662+0000","last_change":"2023-03-25T01:48:57.404560+0000","last_active":"2023-03-25T01:49:12.193662+0000","last_peered":"2023-03-25T01:49:12.193662+0000","last_clean":"2023-03-25T01:49:12.193662+0000","last_became_active":"2023-03-25T01:48:57.404333+0000","last_became_peered":"2023-03-25T01:48:57.404333+0000","last_unstale":"2023-03-25T01:49:12.193662+0000","last_undegraded":"2023-03-25T01:49:12.193662+0000","last_fullsized":"2023-03-25T01:49:12.193662+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T01:48:55.982031+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T01:48:55.982031+0000","last_clean_scrub_stamp":"2023-03-25T01:48:55.982031+0000","objects_scrubbed":0,"log_size":64,"log_dups_size":0,"ondisk_log_size":64,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:18:38.285450+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":970000,"num_objects":3,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":64,"num_write_kb":978,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":3,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2],"acting":[4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":10,"reported_epoch":19,"state":"active+clean","last_fresh":"2023-03-25T01:48:58.909182+0000","last_change":"2023-03-25T01:48:56.989694+0000","last_active":"2023-03-25T01:48:58.909182+0000","last_peered":"2023-03-25T01:48:58.909182+0000","last_clean":"2023-03-25T01:48:58.909182+0000","last_became_active":"2023-03-25T01:48:56.989461+0000","last_became_peered":"2023-03-25T01:48:56.989461+0000","last_unstale":"2023-03-25T01:48:58.909182+0000","last_undegraded":"2023-03-25T01:48:58.909182+0000","last_fullsized":"2023-03-25T01:48:58.909182+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T01:48:55.982031+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T01:48:55.982031+0000","last_clean_scrub_stamp":"2023-03-25T01:48:55.982031+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T12:21:35.179737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":14,"reported_epoch":19,"state":"active+clean","last_fresh":"2023-03-25T01:48:56.988435+0000","last_change":"2023-03-25T01:48:47.347051+0000","last_active":"2023-03-25T01:48:56.988435+0000","last_peered":"2023-03-25T01:48:56.988435+0000","last_clean":"2023-03-25T01:48:56.988435+0000","last_became_active":"2023-03-25T01:48:47.346812+0000","last_became_peered":"2023-03-25T01:48:47.346812+0000","last_unstale":"2023-03-25T01:48:56.988435+0000","last_undegraded":"2023-03-25T01:48:56.988435+0000","last_fullsized":"2023-03-25T01:48:56.988435+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T01:48:45.924614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T01:48:45.924614+0000","last_clean_scrub_stamp":"2023-03-25T01:48:45.924614+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T07:21:10.392042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"20'30","reported_seq":40,"reported_epoch":20,"state":"active+clean","last_fresh":"2023-03-25T01:49:04.130697+0000","last_change":"2023-03-25T01:48:56.990522+0000","last_active":"2023-03-25T01:49:04.130697+0000","last_peered":"2023-03-25T01:49:04.130697+0000","last_clean":"2023-03-25T01:49:04.130697+0000","last_became_active":"2023-03-25T01:48:56.990276+0000","last_became_peered":"2023-03-25T01:48:56.990276+0000","last_unstale":"2023-03-25T01:49:04.130697+0000","last_undegraded":"2023-03-25T01:49:04.130697+0000","last_fullsized":"2023-03-25T01:49:04.130697+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T01:48:55.982031+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T01:48:55.982031+0000","last_clean_scrub_stamp":"2023-03-25T01:48:55.982031+0000","objects_scrubbed":0,"log_size":30,"log_dups_size":0,"ondisk_log_size":30,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:43:07.120795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":650000,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":30,"num_write_kb":649,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":15,"reported_epoch":19,"state":"active+clean","last_fresh":"2023-03-25T01:48:56.988086+0000","last_change":"2023-03-25T01:48:46.941510+0000","last_active":"2023-03-25T01:48:56.988086+0000","last_peered":"2023-03-25T01:48:56.988086+0000","last_clean":"2023-03-25T01:48:56.988086+0000","last_became_active":"2023-03-25T01:48:46.941266+0000","last_became_peered":"2023-03-25T01:48:46.941266+0000","last_unstale":"2023-03-25T01:48:56.988086+0000","last_undegraded":"2023-03-25T01:48:56.988086+0000","last_fullsized":"2023-03-25T01:48:56.988086+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T01:48:45.924614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T01:48:45.924614+0000","last_clean_scrub_stamp":"2023-03-25T01:48:45.924614+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:34:02.770586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1],"acting":[5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":10,"reported_epoch":19,"state":"active+clean","last_fresh":"2023-03-25T01:48:58.909220+0000","last_change":"2023-03-25T01:48:56.990419+0000","last_active":"2023-03-25T01:48:58.909220+0000","last_peered":"2023-03-25T01:48:58.909220+0000","last_clean":"2023-03-25T01:48:58.909220+0000","last_became_active":"2023-03-25T01:48:56.990188+0000","last_became_peered":"2023-03-25T01:48:56.990188+0000","last_unstale":"2023-03-25T01:48:58.909220+0000","last_undegraded":"2023-03-25T01:48:58.909220+0000","last_fullsized":"2023-03-25T01:48:58.909220+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T01:48:55.982031+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T01:48:55.982031+0000","last_clean_scrub_stamp":"2023-03-25T01:48:55.982031+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:43:07.120795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":14,"reported_epoch":19,"state":"active+clean","last_fresh":"2023-03-25T01:48:57.402752+0000","last_change":"2023-03-25T01:48:46.942191+0000","last_active":"2023-03-25T01:48:57.402752+0000","last_peered":"2023-03-25T01:48:57.402752+0000","last_clean":"2023-03-25T01:48:57.402752+0000","last_became_active":"2023-03-25T01:48:46.941918+0000","last_became_peered":"2023-03-25T01:48:46.941918+0000","last_unstale":"2023-03-25T01:48:57.402752+0000","last_undegraded":"2023-03-25T01:48:57.402752+0000","last_fullsized":"2023-03-25T01:48:57.402752+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T01:48:45.924614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T01:48:45.924614+0000","last_clean_scrub_stamp":"2023-03-25T01:48:45.924614+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T10:36:32.544700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5],"acting":[2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":10,"reported_epoch":20,"state":"active+clean","last_fresh":"2023-03-25T01:48:57.985996+0000","last_change":"2023-03-25T01:48:57.404656+0000","last_active":"2023-03-25T01:48:57.985996+0000","last_peered":"2023-03-25T01:48:57.985996+0000","last_clean":"2023-03-25T01:48:57.985996+0000","last_became_active":"2023-03-25T01:48:57.404416+0000","last_became_peered":"2023-03-25T01:48:57.404416+0000","last_unstale":"2023-03-25T01:48:57.985996+0000","last_undegraded":"2023-03-25T01:48:57.985996+0000","last_fullsized":"2023-03-25T01:48:57.985996+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T01:48:55.982031+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T01:48:55.982031+0000","last_clean_scrub_stamp":"2023-03-25T01:48:55.982031+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T12:21:35.179737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":15,"reported_epoch":19,"state":"active+clean","last_fresh":"2023-03-25T01:48:56.987983+0000","last_change":"2023-03-25T01:48:47.283824+0000","last_active":"2023-03-25T01:48:56.987983+0000","last_peered":"2023-03-25T01:48:56.987983+0000","last_clean":"2023-03-25T01:48:56.987983+0000","last_became_active":"2023-03-25T01:48:47.283534+0000","last_became_peered":"2023-03-25T01:48:47.283534+0000","last_unstale":"2023-03-25T01:48:56.987983+0000","last_undegraded":"2023-03-25T01:48:56.987983+0000","last_fullsized":"2023-03-25T01:48:56.987983+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T01:48:45.924614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T01:48:45.924614+0000","last_clean_scrub_stamp":"2023-03-25T01:48:45.924614+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T10:36:32.544700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4],"acting":[5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"13'97","reported_seq":142,"reported_epoch":19,"state":"active+clean","last_fresh":"2023-03-25T01:48:56.988118+0000","last_change":"2023-03-25T01:48:45.934080+0000","last_active":"2023-03-25T01:48:56.988118+0000","last_peered":"2023-03-25T01:48:56.988118+0000","last_clean":"2023-03-25T01:48:56.988118+0000","last_became_active":"2023-03-25T01:48:45.933746+0000","last_became_peered":"2023-03-25T01:48:45.933746+0000","last_unstale":"2023-03-25T01:48:56.988118+0000","last_undegraded":"2023-03-25T01:48:56.988118+0000","last_fullsized":"2023-03-25T01:48:56.988118+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T01:48:44.920729+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T01:48:44.920729+0000","last_clean_scrub_stamp":"2023-03-25T01:48:44.920729+0000","objects_scrubbed":0,"log_size":97,"log_dups_size":0,"ondisk_log_size":97,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:43:52.243010+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":137,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.2","version":"20'30","reported_seq":40,"reported_epoch":20,"state":"active+clean","last_fresh":"2023-03-25T01:49:06.431894+0000","last_change":"2023-03-25T01:48:56.991149+0000","last_active":"2023-03-25T01:49:06.431894+0000","last_peered":"2023-03-25T01:49:06.431894+0000","last_clean":"2023-03-25T01:49:06.431894+0000","last_became_active":"2023-03-25T01:48:56.990889+0000","last_became_peered":"2023-03-25T01:48:56.990889+0000","last_unstale":"2023-03-25T01:49:06.431894+0000","last_undegraded":"2023-03-25T01:49:06.431894+0000","last_fullsized":"2023-03-25T01:49:06.431894+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T01:48:55.982031+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T01:48:55.982031+0000","last_clean_scrub_stamp":"2023-03-25T01:48:55.982031+0000","objects_scrubbed":0,"log_size":30,"log_dups_size":0,"ondisk_log_size":30,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T10:51:06.108055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":650000,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":30,"num_write_kb":649,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5],"acting":[3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":14,"reported_epoch":19,"state":"active+clean","last_fresh":"2023-03-25T01:48:56.984507+0000","last_change":"2023-03-25T01:48:46.943089+0000","last_active":"2023-03-25T01:48:56.984507+0000","last_peered":"2023-03-25T01:48:56.984507+0000","last_clean":"2023-03-25T01:48:56.984507+0000","last_became_active":"2023-03-25T01:48:46.942843+0000","last_became_peered":"2023-03-25T01:48:46.942843+0000","last_unstale":"2023-03-25T01:48:56.984507+0000","last_undegraded":"2023-03-25T01:48:56.984507+0000","last_fullsized":"2023-03-25T01:48:56.984507+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T01:48:45.924614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T01:48:45.924614+0000","last_clean_scrub_stamp":"2023-03-25T01:48:45.924614+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T07:21:10.392042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5],"acting":[3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1","version":"20'34","reported_seq":44,"reported_epoch":20,"state":"active+clean","last_fresh":"2023-03-25T01:49:13.951565+0000","last_change":"2023-03-25T01:48:56.989334+0000","last_active":"2023-03-25T01:49:13.951565+0000","last_peered":"2023-03-25T01:49:13.951565+0000","last_clean":"2023-03-25T01:49:13.951565+0000","last_became_active":"2023-03-25T01:48:56.989079+0000","last_became_peered":"2023-03-25T01:48:56.989079+0000","last_unstale":"2023-03-25T01:49:13.951565+0000","last_undegraded":"2023-03-25T01:49:13.951565+0000","last_fullsized":"2023-03-25T01:49:13.951565+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T01:48:55.982031+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T01:48:55.982031+0000","last_clean_scrub_stamp":"2023-03-25T01:48:55.982031+0000","objects_scrubbed":0,"log_size":34,"log_dups_size":0,"ondisk_log_size":34,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T07:51:55.253730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":330000,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":34,"num_write_kb":339,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":14,"reported_epoch":19,"state":"active+clean","last_fresh":"2023-03-25T01:48:56.984518+0000","last_change":"2023-03-25T01:48:47.283615+0000","last_active":"2023-03-25T01:48:56.984518+0000","last_peered":"2023-03-25T01:48:56.984518+0000","last_clean":"2023-03-25T01:48:56.984518+0000","last_became_active":"2023-03-25T01:48:47.283359+0000","last_became_peered":"2023-03-25T01:48:47.283359+0000","last_unstale":"2023-03-25T01:48:56.984518+0000","last_undegraded":"2023-03-25T01:48:56.984518+0000","last_fullsized":"2023-03-25T01:48:56.984518+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T01:48:45.924614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T01:48:45.924614+0000","last_clean_scrub_stamp":"2023-03-25T01:48:45.924614+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T02:55:58.247062+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.4","version":"20'5","reported_seq":15,"reported_epoch":20,"state":"active+clean","last_fresh":"2023-03-25T01:48:58.920001+0000","last_change":"2023-03-25T01:48:57.404723+0000","last_active":"2023-03-25T01:48:58.920001+0000","last_peered":"2023-03-25T01:48:58.920001+0000","last_clean":"2023-03-25T01:48:58.920001+0000","last_became_active":"2023-03-25T01:48:57.404409+0000","last_became_peered":"2023-03-25T01:48:57.404409+0000","last_unstale":"2023-03-25T01:48:58.920001+0000","last_undegraded":"2023-03-25T01:48:58.920001+0000","last_fullsized":"2023-03-25T01:48:58.920001+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T01:48:55.982031+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T01:48:55.982031+0000","last_clean_scrub_stamp":"2023-03-25T01:48:55.982031+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T07:51:55.253730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":640000,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5,"num_write_kb":627,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"20'88","reported_seq":98,"reported_epoch":20,"state":"active+clean","last_fresh":"2023-03-25T01:49:18.814701+0000","last_change":"2023-03-25T01:48:56.988823+0000","last_active":"2023-03-25T01:49:18.814701+0000","last_peered":"2023-03-25T01:49:18.814701+0000","last_clean":"2023-03-25T01:49:18.814701+0000","last_became_active":"2023-03-25T01:48:56.988568+0000","last_became_peered":"2023-03-25T01:48:56.988568+0000","last_unstale":"2023-03-25T01:49:18.814701+0000","last_undegraded":"2023-03-25T01:49:18.814701+0000","last_fullsized":"2023-03-25T01:49:18.814701+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T01:48:55.982031+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T01:48:55.982031+0000","last_clean_scrub_stamp":"2023-03-25T01:48:55.982031+0000","objects_scrubbed":0,"log_size":88,"log_dups_size":0,"ondisk_log_size":88,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T10:51:06.108055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1310000,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":88,"num_write_kb":1314,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":14,"reported_epoch":19,"state":"active+clean","last_fresh":"2023-03-25T01:48:56.988499+0000","last_change":"2023-03-25T01:48:47.347152+0000","last_active":"2023-03-25T01:48:56.988499+0000","last_peered":"2023-03-25T01:48:56.988499+0000","last_clean":"2023-03-25T01:48:56.988499+0000","last_became_active":"2023-03-25T01:48:47.346919+0000","last_became_peered":"2023-03-25T01:48:47.346919+0000","last_unstale":"2023-03-25T01:48:56.988499+0000","last_undegraded":"2023-03-25T01:48:56.988499+0000","last_fullsized":"2023-03-25T01:48:56.988499+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T01:48:45.924614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T01:48:45.924614+0000","last_clean_scrub_stamp":"2023-03-25T01:48:45.924614+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T02:55:58.247062+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":14,"reported_epoch":19,"state":"active+clean","last_fresh":"2023-03-25T01:48:56.984528+0000","last_change":"2023-03-25T01:48:47.347110+0000","last_active":"2023-03-25T01:48:56.984528+0000","last_peered":"2023-03-25T01:48:56.984528+0000","last_clean":"2023-03-25T01:48:56.984528+0000","last_became_active":"2023-03-25T01:48:47.346832+0000","last_became_peered":"2023-03-25T01:48:47.346832+0000","last_unstale":"2023-03-25T01:48:56.984528+0000","last_undegraded":"2023-03-25T01:48:56.984528+0000","last_fullsized":"2023-03-25T01:48:56.984528+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T01:48:45.924614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T01:48:45.924614+0000","last_clean_scrub_stamp":"2023-03-25T01:48:45.924614+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:34:02.770586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"20'126","reported_seq":136,"reported_epoch":20,"state":"active+clean","last_fresh":"2023-03-25T01:49:17.626037+0000","last_change":"2023-03-25T01:48:56.990341+0000","last_active":"2023-03-25T01:49:17.626037+0000","last_peered":"2023-03-25T01:49:17.626037+0000","last_clean":"2023-03-25T01:49:17.626037+0000","last_became_active":"2023-03-25T01:48:56.990039+0000","last_became_peered":"2023-03-25T01:48:56.990039+0000","last_unstale":"2023-03-25T01:49:17.626037+0000","last_undegraded":"2023-03-25T01:49:17.626037+0000","last_fullsized":"2023-03-25T01:49:17.626037+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T01:48:55.982031+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T01:48:55.982031+0000","last_clean_scrub_stamp":"2023-03-25T01:48:55.982031+0000","objects_scrubbed":0,"log_size":126,"log_dups_size":0,"ondisk_log_size":126,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T06:14:10.792416+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1630000,"num_objects":5,"num_object_clones":0,"num_object_copies":10,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":126,"num_write_kb":1653,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":5,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":3,"num_pg":12,"stat_sum":{"num_bytes":6510000,"num_objects":20,"num_object_clones":0,"num_object_copies":40,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":20,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":403,"num_write_kb":6544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":19,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":13123584,"data_stored":13020000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":403,"ondisk_log_size":403,"up":24,"acting":24,"num_store_stats":6},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":137,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":97,"ondisk_log_size":97,"up":2,"acting":2,"num_store_stats":2},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":6}],"osd_stats":[{"osd":5,"up_from":11,"seq":47244640265,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":104857600,"kb_used":6608,"kb_used_data":1480,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":104850992,"statfs":{"total":107374182400,"available":107367415808,"internally_reserved":0,"allocated":1515520,"data_stored":1366284,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,3,4],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34999999999999998}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.307}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29399999999999998}]},{"osd":3,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.434}]},{"osd":4,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.36499999999999999}]}]},{"osd":4,"up_from":11,"seq":47244640267,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":104857600,"kb_used":8024,"kb_used_data":2768,"kb_used_omap":0,"kb_used_meta":5248,"kb_avail":104849576,"statfs":{"total":107374182400,"available":107365965824,"internally_reserved":0,"allocated":2834432,"data_stored":2676284,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5373952},"hb_peers":[0,1,2,3,5],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.31900000000000001}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.24099999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.25800000000000001}]},{"osd":3,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.41499999999999998}]},{"osd":5,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.435}]}]},{"osd":3,"up_from":11,"seq":47244640266,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":104857600,"kb_used":8736,"kb_used_data":3416,"kb_used_omap":0,"kb_used_meta":5312,"kb_avail":104848864,"statfs":{"total":107374182400,"available":107365236736,"internally_reserved":0,"allocated":3497984,"data_stored":3336284,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5439488},"hb_peers":[0,1,2,4,5],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.30199999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.35599999999999998}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.38400000000000001}]},{"osd":4,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.312}]},{"osd":5,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.32000000000000001}]}]},{"osd":2,"up_from":11,"seq":47244640266,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":104857600,"kb_used":6916,"kb_used_data":1788,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":104850684,"statfs":{"total":107374182400,"available":107367100416,"internally_reserved":0,"allocated":1830912,"data_stored":1676284,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,3,4,5],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.498}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.45200000000000001}]},{"osd":3,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.52800000000000002}]},{"osd":4,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.32100000000000001}]},{"osd":5,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.58299999999999996}]}]},{"osd":0,"up_from":11,"seq":47244640265,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":104857600,"kb_used":8348,"kb_used_data":2708,"kb_used_omap":0,"kb_used_meta":5632,"kb_avail":104849252,"statfs":{"total":107374182400,"available":107365634048,"internally_reserved":0,"allocated":2772992,"data_stored":2616652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5767168},"hb_peers":[1,2,3,4,5],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.32800000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26000000000000001}]},{"osd":3,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.30399999999999999}]},{"osd":4,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.24399999999999999}]},{"osd":5,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":1,"up_from":11,"seq":47244640266,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":104857600,"kb_used":8656,"kb_used_data":3016,"kb_used_omap":0,"kb_used_meta":5632,"kb_avail":104848944,"statfs":{"total":107374182400,"available":107365318656,"internally_reserved":0,"allocated":3088384,"data_stored":2926652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5767168},"hb_peers":[0,2,3,4,5],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.27300000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34799999999999998}]},{"osd":3,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.24399999999999999}]},{"osd":4,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.36499999999999999}]},{"osd":5,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.49099999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1974272,"data_stored":1960000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2289664,"data_stored":2270000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1626112,"data_stored":1610000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":3293184,"data_stored":3270000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2629632,"data_stored":2610000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1310720,"data_stored":1300000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T02:01:50.044 INFO:tasks.ceph:pgid 3.b last_scrub_stamp 2023-03-25T01:48:55.982031+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=48, tm_sec=55, tm_wday=5, tm_yday=84, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=49, tm_sec=28, tm_wday=5, tm_yday=84, tm_isdst=0)
, 2023-03-25T02:01:50.044 INFO:tasks.ceph:pgid 3.a last_scrub_stamp 2023-03-25T01:48:55.982031+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=48, tm_sec=55, tm_wday=5, tm_yday=84, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=49, tm_sec=28, tm_wday=5, tm_yday=84, tm_isdst=0)

Ignoring malparsed backtrace: 2023-03-25T02:03:11.095 DEBUG:teuthology.orchestra.run.smithi087:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T02:03:11.419 INFO:teuthology.orchestra.run.smithi087.stdout:
, dumped all

Ignoring malparsed backtrace: 2023-03-25T02:03:58.964 INFO:tasks.ceph:pgid 3.8 last_scrub_stamp 2023-03-25T01:48:55.982031+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=48, tm_sec=55, tm_wday=5, tm_yday=84, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=49, tm_sec=28, tm_wday=5, tm_yday=84, tm_isdst=0)
, 2023-03-25T02:03:58.964 INFO:tasks.ceph:pgid 2.4 last_scrub_stamp 2023-03-25T01:48:45.924614+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=48, tm_sec=45, tm_wday=5, tm_yday=84, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=49, tm_sec=28, tm_wday=5, tm_yday=84, tm_isdst=0)
, 2023-03-25T02:03:58.964 INFO:tasks.ceph:pgid 3.5 last_scrub_stamp 2023-03-25T01:48:55.982031+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=48, tm_sec=55, tm_wday=5, tm_yday=84, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=49, tm_sec=28, tm_wday=5, tm_yday=84, tm_isdst=0)

Ignoring malparsed backtrace: 2023-03-25T02:05:20.356 INFO:tasks.ceph:pgid 3.b last_scrub_stamp 2023-03-25T01:48:55.982031+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=48, tm_sec=55, tm_wday=5, tm_yday=84, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=49, tm_sec=28, tm_wday=5, tm_yday=84, tm_isdst=0)
, 2023-03-25T02:05:20.356 INFO:tasks.ceph:pgid 3.a last_scrub_stamp 2023-03-25T01:48:55.982031+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=48, tm_sec=55, tm_wday=5, tm_yday=84, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=49, tm_sec=28, tm_wday=5, tm_yday=84, tm_isdst=0)
, 2023-03-25T02:05:20.356 INFO:tasks.ceph:pgid 3.9 last_scrub_stamp 2023-03-25T01:48:55.982031+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=48, tm_sec=55, tm_wday=5, tm_yday=84, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=49, tm_sec=28, tm_wday=5, tm_yday=84, tm_isdst=0)

Ignoring malparsed backtrace: 2023-03-25T02:06:07.907 INFO:tasks.ceph:pgid 3.3 last_scrub_stamp 2023-03-25T01:48:55.982031+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=48, tm_sec=55, tm_wday=5, tm_yday=84, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=49, tm_sec=28, tm_wday=5, tm_yday=84, tm_isdst=0)
, 2023-03-25T02:06:07.907 INFO:tasks.ceph:pgid 2.1 last_scrub_stamp 2023-03-25T01:48:45.924614+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=48, tm_sec=45, tm_wday=5, tm_yday=84, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=49, tm_sec=28, tm_wday=5, tm_yday=84, tm_isdst=0)
, 2023-03-25T02:06:07.907 INFO:tasks.ceph:pgid 3.0 last_scrub_stamp 2023-03-25T01:48:55.982031+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=48, tm_sec=55, tm_wday=5, tm_yday=84, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=49, tm_sec=28, tm_wday=5, tm_yday=84, tm_isdst=0)

Ignoring malparsed backtrace: 2023-03-25T02:07:29.309 INFO:tasks.ceph:pgid 2.4 last_scrub_stamp 2023-03-25T01:48:45.924614+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=48, tm_sec=45, tm_wday=5, tm_yday=84, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=49, tm_sec=28, tm_wday=5, tm_yday=84, tm_isdst=0)
, 2023-03-25T02:07:29.309 INFO:tasks.ceph:pgid 3.5 last_scrub_stamp 2023-03-25T01:48:55.982031+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=48, tm_sec=55, tm_wday=5, tm_yday=84, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=49, tm_sec=28, tm_wday=5, tm_yday=84, tm_isdst=0)
, 2023-03-25T02:07:29.309 INFO:tasks.ceph:pgid 2.2 last_scrub_stamp 2023-03-25T01:48:45.924614+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=48, tm_sec=45, tm_wday=5, tm_yday=84, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=49, tm_sec=28, tm_wday=5, tm_yday=84, tm_isdst=0)

Ignoring malparsed backtrace: 2023-03-25T02:08:16.851 INFO:tasks.ceph:pgid 1.0 last_scrub_stamp 2023-03-25T01:48:44.920729+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=48, tm_sec=44, tm_wday=5, tm_yday=84, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=49, tm_sec=28, tm_wday=5, tm_yday=84, tm_isdst=0)
, 2023-03-25T02:08:16.851 INFO:tasks.ceph:pgid 3.2 last_scrub_stamp 2023-03-25T01:48:55.982031+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=48, tm_sec=55, tm_wday=5, tm_yday=84, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=49, tm_sec=28, tm_wday=5, tm_yday=84, tm_isdst=0)
, 2023-03-25T02:08:16.852 INFO:tasks.ceph:pgid 2.0 last_scrub_stamp 2023-03-25T01:48:45.924614+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=48, tm_sec=45, tm_wday=5, tm_yday=84, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=49, tm_sec=28, tm_wday=5, tm_yday=84, tm_isdst=0)

Ignoring malparsed backtrace: 2023-03-25T02:09:38.242 INFO:tasks.ceph:pgid 2.1 last_scrub_stamp 2023-03-25T01:48:45.924614+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=48, tm_sec=45, tm_wday=5, tm_yday=84, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=49, tm_sec=28, tm_wday=5, tm_yday=84, tm_isdst=0)
, 2023-03-25T02:09:38.242 INFO:tasks.ceph:pgid 3.0 last_scrub_stamp 2023-03-25T01:48:55.982031+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=48, tm_sec=55, tm_wday=5, tm_yday=84, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=49, tm_sec=28, tm_wday=5, tm_yday=84, tm_isdst=0)
, 2023-03-25T02:09:38.242 INFO:tasks.ceph:pgid 2.3 last_scrub_stamp 2023-03-25T01:48:45.924614+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=48, tm_sec=45, tm_wday=5, tm_yday=84, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=49, tm_sec=28, tm_wday=5, tm_yday=84, tm_isdst=0)

Ignoring malparsed backtrace: 2023-03-25T02:10:25.793 INFO:tasks.ceph:pgid 2.5 last_scrub_stamp 2023-03-25T01:48:45.924614+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=48, tm_sec=45, tm_wday=5, tm_yday=84, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=49, tm_sec=28, tm_wday=5, tm_yday=84, tm_isdst=0)
, 2023-03-25T02:10:25.793 INFO:tasks.ceph:pgid 3.4 last_scrub_stamp 2023-03-25T01:48:55.982031+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=48, tm_sec=55, tm_wday=5, tm_yday=84, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=49, tm_sec=28, tm_wday=5, tm_yday=84, tm_isdst=0)
, 2023-03-25T02:10:25.793 INFO:tasks.ceph:pgid 3.7 last_scrub_stamp 2023-03-25T01:48:55.982031+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=48, tm_sec=55, tm_wday=5, tm_yday=84, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=49, tm_sec=28, tm_wday=5, tm_yday=84, tm_isdst=0)

Ignoring malparsed backtrace: 2023-03-25T02:11:47.190 INFO:tasks.ceph:pgid 3.2 last_scrub_stamp 2023-03-25T01:48:55.982031+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=48, tm_sec=55, tm_wday=5, tm_yday=84, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=49, tm_sec=28, tm_wday=5, tm_yday=84, tm_isdst=0)
, 2023-03-25T02:11:47.190 INFO:tasks.ceph:pgid 2.0 last_scrub_stamp 2023-03-25T01:48:45.924614+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=48, tm_sec=45, tm_wday=5, tm_yday=84, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=49, tm_sec=28, tm_wday=5, tm_yday=84, tm_isdst=0)
, 2023-03-25T02:11:47.190 INFO:tasks.ceph:pgid 3.1 last_scrub_stamp 2023-03-25T01:48:55.982031+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=48, tm_sec=55, tm_wday=5, tm_yday=84, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=49, tm_sec=28, tm_wday=5, tm_yday=84, tm_isdst=0)

Ignoring malparsed backtrace: 2023-03-25T02:12:34.721 INFO:tasks.ceph:pgid 2.7 last_scrub_stamp 2023-03-25T01:48:45.924614+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=48, tm_sec=45, tm_wday=5, tm_yday=84, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=49, tm_sec=28, tm_wday=5, tm_yday=84, tm_isdst=0)
, 2023-03-25T02:12:34.722 INFO:tasks.ceph:pgid 3.6 last_scrub_stamp 2023-03-25T01:48:55.982031+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=48, tm_sec=55, tm_wday=5, tm_yday=84, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=49, tm_sec=28, tm_wday=5, tm_yday=84, tm_isdst=0)
, 2023-03-25T02:12:34.722 INFO:tasks.ceph:Still waiting for all pgs to be scrubbed.

Ignoring malparsed backtrace: 2023-03-25T02:13:56.118 INFO:tasks.ceph:pgid 3.4 last_scrub_stamp 2023-03-25T01:48:55.982031+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=48, tm_sec=55, tm_wday=5, tm_yday=84, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=49, tm_sec=28, tm_wday=5, tm_yday=84, tm_isdst=0)
, 2023-03-25T02:13:56.119 INFO:tasks.ceph:pgid 3.7 last_scrub_stamp 2023-03-25T01:48:55.982031+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=48, tm_sec=55, tm_wday=5, tm_yday=84, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=49, tm_sec=28, tm_wday=5, tm_yday=84, tm_isdst=0)
, 2023-03-25T02:13:56.119 INFO:tasks.ceph:pgid 2.6 last_scrub_stamp 2023-03-25T01:48:45.924614+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=48, tm_sec=45, tm_wday=5, tm_yday=84, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=49, tm_sec=28, tm_wday=5, tm_yday=84, tm_isdst=0)

Ignoring malparsed backtrace: 2023-03-25T02:15:03.973 INFO:teuthology.orchestra.run.smithi087.stdout:
, dumped all
, 2023-03-25T02:15:03.986 INFO:teuthology.orchestra.run.smithi087.stdout:{"pg_ready":true,"pg_map":{"version":805,"stamp":"2023-03-25T02:15:03.017634+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":7100368,"num_objects":22,"num_object_clones":0,"num_object_copies":44,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":540,"num_write_kb":8438,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":19,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":500,"ondisk_log_size":500,"up":42,"acting":42,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":42,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":629145600,"kb_used":47288,"kb_used_data":15176,"kb_used_omap":0,"kb_used_meta":32064,"kb_avail":629098312,"statfs":{"total":644245094400,"available":644196671488,"internally_reserved":0,"allocated":15540224,"data_stored":14598440,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":32833536},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002272"},"pg_stats":[{"pgid":"3.b","version":"20'26","reported_seq":36,"reported_epoch":20,"state":"active+clean","last_fresh":"2023-03-25T01:49:07.722753+0000","last_change":"2023-03-25T01:48:56.990468+0000","last_active":"2023-03-25T01:49:07.722753+0000","last_peered":"2023-03-25T01:49:07.722753+0000","last_clean":"2023-03-25T01:49:07.722753+0000","last_became_active":"2023-03-25T01:48:56.990194+0000","last_became_peered":"2023-03-25T01:48:56.990194+0000","last_unstale":"2023-03-25T01:49:07.722753+0000","last_undegraded":"2023-03-25T01:49:07.722753+0000","last_fullsized":"2023-03-25T01:49:07.722753+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T01:48:55.982031+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T01:48:55.982031+0000","last_clean_scrub_stamp":"2023-03-25T01:48:55.982031+0000","objects_scrubbed":0,"log_size":26,"log_dups_size":0,"ondisk_log_size":26,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T07:51:55.253730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":330000,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":26,"num_write_kb":335,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":10,"reported_epoch":19,"state":"active+clean","last_fresh":"2023-03-25T01:48:58.910560+0000","last_change":"2023-03-25T01:48:56.990354+0000","last_active":"2023-03-25T01:48:58.910560+0000","last_peered":"2023-03-25T01:48:58.910560+0000","last_clean":"2023-03-25T01:48:58.910560+0000","last_became_active":"2023-03-25T01:48:56.989754+0000","last_became_peered":"2023-03-25T01:48:56.989754+0000","last_unstale":"2023-03-25T01:48:58.910560+0000","last_undegraded":"2023-03-25T01:48:58.910560+0000","last_fullsized":"2023-03-25T01:48:58.910560+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T01:48:55.982031+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T01:48:55.982031+0000","last_clean_scrub_stamp":"2023-03-25T01:48:55.982031+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:18:38.285450+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"20'64","reported_seq":74,"reported_epoch":20,"state":"active+clean","last_fresh":"2023-03-25T01:49:12.193662+0000","last_change":"2023-03-25T01:48:57.404560+0000","last_active":"2023-03-25T01:49:12.193662+0000","last_peered":"2023-03-25T01:49:12.193662+0000","last_clean":"2023-03-25T01:49:12.193662+0000","last_became_active":"2023-03-25T01:48:57.404333+0000","last_became_peered":"2023-03-25T01:48:57.404333+0000","last_unstale":"2023-03-25T01:49:12.193662+0000","last_undegraded":"2023-03-25T01:49:12.193662+0000","last_fullsized":"2023-03-25T01:49:12.193662+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T01:48:55.982031+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T01:48:55.982031+0000","last_clean_scrub_stamp":"2023-03-25T01:48:55.982031+0000","objects_scrubbed":0,"log_size":64,"log_dups_size":0,"ondisk_log_size":64,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:18:38.285450+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":970000,"num_objects":3,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":64,"num_write_kb":978,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":3,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2],"acting":[4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":10,"reported_epoch":19,"state":"active+clean","last_fresh":"2023-03-25T01:48:58.909182+0000","last_change":"2023-03-25T01:48:56.989694+0000","last_active":"2023-03-25T01:48:58.909182+0000","last_peered":"2023-03-25T01:48:58.909182+0000","last_clean":"2023-03-25T01:48:58.909182+0000","last_became_active":"2023-03-25T01:48:56.989461+0000","last_became_peered":"2023-03-25T01:48:56.989461+0000","last_unstale":"2023-03-25T01:48:58.909182+0000","last_undegraded":"2023-03-25T01:48:58.909182+0000","last_fullsized":"2023-03-25T01:48:58.909182+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T01:48:55.982031+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T01:48:55.982031+0000","last_clean_scrub_stamp":"2023-03-25T01:48:55.982031+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T12:21:35.179737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":14,"reported_epoch":19,"state":"active+clean","last_fresh":"2023-03-25T01:48:56.988435+0000","last_change":"2023-03-25T01:48:47.347051+0000","last_active":"2023-03-25T01:48:56.988435+0000","last_peered":"2023-03-25T01:48:56.988435+0000","last_clean":"2023-03-25T01:48:56.988435+0000","last_became_active":"2023-03-25T01:48:47.346812+0000","last_became_peered":"2023-03-25T01:48:47.346812+0000","last_unstale":"2023-03-25T01:48:56.988435+0000","last_undegraded":"2023-03-25T01:48:56.988435+0000","last_fullsized":"2023-03-25T01:48:56.988435+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T01:48:45.924614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T01:48:45.924614+0000","last_clean_scrub_stamp":"2023-03-25T01:48:45.924614+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T07:21:10.392042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"20'30","reported_seq":40,"reported_epoch":20,"state":"active+clean","last_fresh":"2023-03-25T01:49:04.130697+0000","last_change":"2023-03-25T01:48:56.990522+0000","last_active":"2023-03-25T01:49:04.130697+0000","last_peered":"2023-03-25T01:49:04.130697+0000","last_clean":"2023-03-25T01:49:04.130697+0000","last_became_active":"2023-03-25T01:48:56.990276+0000","last_became_peered":"2023-03-25T01:48:56.990276+0000","last_unstale":"2023-03-25T01:49:04.130697+0000","last_undegraded":"2023-03-25T01:49:04.130697+0000","last_fullsized":"2023-03-25T01:49:04.130697+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T01:48:55.982031+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T01:48:55.982031+0000","last_clean_scrub_stamp":"2023-03-25T01:48:55.982031+0000","objects_scrubbed":0,"log_size":30,"log_dups_size":0,"ondisk_log_size":30,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:43:07.120795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":650000,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":30,"num_write_kb":649,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":15,"reported_epoch":19,"state":"active+clean","last_fresh":"2023-03-25T01:48:56.988086+0000","last_change":"2023-03-25T01:48:46.941510+0000","last_active":"2023-03-25T01:48:56.988086+0000","last_peered":"2023-03-25T01:48:56.988086+0000","last_clean":"2023-03-25T01:48:56.988086+0000","last_became_active":"2023-03-25T01:48:46.941266+0000","last_became_peered":"2023-03-25T01:48:46.941266+0000","last_unstale":"2023-03-25T01:48:56.988086+0000","last_undegraded":"2023-03-25T01:48:56.988086+0000","last_fullsized":"2023-03-25T01:48:56.988086+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T01:48:45.924614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T01:48:45.924614+0000","last_clean_scrub_stamp":"2023-03-25T01:48:45.924614+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:34:02.770586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1],"acting":[5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":10,"reported_epoch":19,"state":"active+clean","last_fresh":"2023-03-25T01:48:58.909220+0000","last_change":"2023-03-25T01:48:56.990419+0000","last_active":"2023-03-25T01:48:58.909220+0000","last_peered":"2023-03-25T01:48:58.909220+0000","last_clean":"2023-03-25T01:48:58.909220+0000","last_became_active":"2023-03-25T01:48:56.990188+0000","last_became_peered":"2023-03-25T01:48:56.990188+0000","last_unstale":"2023-03-25T01:48:58.909220+0000","last_undegraded":"2023-03-25T01:48:58.909220+0000","last_fullsized":"2023-03-25T01:48:58.909220+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T01:48:55.982031+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T01:48:55.982031+0000","last_clean_scrub_stamp":"2023-03-25T01:48:55.982031+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:43:07.120795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":14,"reported_epoch":19,"state":"active+clean","last_fresh":"2023-03-25T01:48:57.402752+0000","last_change":"2023-03-25T01:48:46.942191+0000","last_active":"2023-03-25T01:48:57.402752+0000","last_peered":"2023-03-25T01:48:57.402752+0000","last_clean":"2023-03-25T01:48:57.402752+0000","last_became_active":"2023-03-25T01:48:46.941918+0000","last_became_peered":"2023-03-25T01:48:46.941918+0000","last_unstale":"2023-03-25T01:48:57.402752+0000","last_undegraded":"2023-03-25T01:48:57.402752+0000","last_fullsized":"2023-03-25T01:48:57.402752+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T01:48:45.924614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T01:48:45.924614+0000","last_clean_scrub_stamp":"2023-03-25T01:48:45.924614+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T10:36:32.544700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5],"acting":[2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":10,"reported_epoch":20,"state":"active+clean","last_fresh":"2023-03-25T01:48:57.985996+0000","last_change":"2023-03-25T01:48:57.404656+0000","last_active":"2023-03-25T01:48:57.985996+0000","last_peered":"2023-03-25T01:48:57.985996+0000","last_clean":"2023-03-25T01:48:57.985996+0000","last_became_active":"2023-03-25T01:48:57.404416+0000","last_became_peered":"2023-03-25T01:48:57.404416+0000","last_unstale":"2023-03-25T01:48:57.985996+0000","last_undegraded":"2023-03-25T01:48:57.985996+0000","last_fullsized":"2023-03-25T01:48:57.985996+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T01:48:55.982031+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T01:48:55.982031+0000","last_clean_scrub_stamp":"2023-03-25T01:48:55.982031+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T12:21:35.179737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":15,"reported_epoch":19,"state":"active+clean","last_fresh":"2023-03-25T01:48:56.987983+0000","last_change":"2023-03-25T01:48:47.283824+0000","last_active":"2023-03-25T01:48:56.987983+0000","last_peered":"2023-03-25T01:48:56.987983+0000","last_clean":"2023-03-25T01:48:56.987983+0000","last_became_active":"2023-03-25T01:48:47.283534+0000","last_became_peered":"2023-03-25T01:48:47.283534+0000","last_unstale":"2023-03-25T01:48:56.987983+0000","last_undegraded":"2023-03-25T01:48:56.987983+0000","last_fullsized":"2023-03-25T01:48:56.987983+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T01:48:45.924614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T01:48:45.924614+0000","last_clean_scrub_stamp":"2023-03-25T01:48:45.924614+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T10:36:32.544700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4],"acting":[5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"13'97","reported_seq":142,"reported_epoch":19,"state":"active+clean","last_fresh":"2023-03-25T01:48:56.988118+0000","last_change":"2023-03-25T01:48:45.934080+0000","last_active":"2023-03-25T01:48:56.988118+0000","last_peered":"2023-03-25T01:48:56.988118+0000","last_clean":"2023-03-25T01:48:56.988118+0000","last_became_active":"2023-03-25T01:48:45.933746+0000","last_became_peered":"2023-03-25T01:48:45.933746+0000","last_unstale":"2023-03-25T01:48:56.988118+0000","last_undegraded":"2023-03-25T01:48:56.988118+0000","last_fullsized":"2023-03-25T01:48:56.988118+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T01:48:44.920729+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T01:48:44.920729+0000","last_clean_scrub_stamp":"2023-03-25T01:48:44.920729+0000","objects_scrubbed":0,"log_size":97,"log_dups_size":0,"ondisk_log_size":97,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:43:52.243010+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":137,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.2","version":"20'30","reported_seq":40,"reported_epoch":20,"state":"active+clean","last_fresh":"2023-03-25T01:49:06.431894+0000","last_change":"2023-03-25T01:48:56.991149+0000","last_active":"2023-03-25T01:49:06.431894+0000","last_peered":"2023-03-25T01:49:06.431894+0000","last_clean":"2023-03-25T01:49:06.431894+0000","last_became_active":"2023-03-25T01:48:56.990889+0000","last_became_peered":"2023-03-25T01:48:56.990889+0000","last_unstale":"2023-03-25T01:49:06.431894+0000","last_undegraded":"2023-03-25T01:49:06.431894+0000","last_fullsized":"2023-03-25T01:49:06.431894+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T01:48:55.982031+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T01:48:55.982031+0000","last_clean_scrub_stamp":"2023-03-25T01:48:55.982031+0000","objects_scrubbed":0,"log_size":30,"log_dups_size":0,"ondisk_log_size":30,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T10:51:06.108055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":650000,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":30,"num_write_kb":649,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5],"acting":[3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":14,"reported_epoch":19,"state":"active+clean","last_fresh":"2023-03-25T01:48:56.984507+0000","last_change":"2023-03-25T01:48:46.943089+0000","last_active":"2023-03-25T01:48:56.984507+0000","last_peered":"2023-03-25T01:48:56.984507+0000","last_clean":"2023-03-25T01:48:56.984507+0000","last_became_active":"2023-03-25T01:48:46.942843+0000","last_became_peered":"2023-03-25T01:48:46.942843+0000","last_unstale":"2023-03-25T01:48:56.984507+0000","last_undegraded":"2023-03-25T01:48:56.984507+0000","last_fullsized":"2023-03-25T01:48:56.984507+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T01:48:45.924614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T01:48:45.924614+0000","last_clean_scrub_stamp":"2023-03-25T01:48:45.924614+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T07:21:10.392042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5],"acting":[3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1","version":"20'34","reported_seq":44,"reported_epoch":20,"state":"active+clean","last_fresh":"2023-03-25T01:49:13.951565+0000","last_change":"2023-03-25T01:48:56.989334+0000","last_active":"2023-03-25T01:49:13.951565+0000","last_peered":"2023-03-25T01:49:13.951565+0000","last_clean":"2023-03-25T01:49:13.951565+0000","last_became_active":"2023-03-25T01:48:56.989079+0000","last_became_peered":"2023-03-25T01:48:56.989079+0000","last_unstale":"2023-03-25T01:49:13.951565+0000","last_undegraded":"2023-03-25T01:49:13.951565+0000","last_fullsized":"2023-03-25T01:49:13.951565+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T01:48:55.982031+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T01:48:55.982031+0000","last_clean_scrub_stamp":"2023-03-25T01:48:55.982031+0000","objects_scrubbed":0,"log_size":34,"log_dups_size":0,"ondisk_log_size":34,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T07:51:55.253730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":330000,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":34,"num_write_kb":339,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":14,"reported_epoch":19,"state":"active+clean","last_fresh":"2023-03-25T01:48:56.984518+0000","last_change":"2023-03-25T01:48:47.283615+0000","last_active":"2023-03-25T01:48:56.984518+0000","last_peered":"2023-03-25T01:48:56.984518+0000","last_clean":"2023-03-25T01:48:56.984518+0000","last_became_active":"2023-03-25T01:48:47.283359+0000","last_became_peered":"2023-03-25T01:48:47.283359+0000","last_unstale":"2023-03-25T01:48:56.984518+0000","last_undegraded":"2023-03-25T01:48:56.984518+0000","last_fullsized":"2023-03-25T01:48:56.984518+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T01:48:45.924614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T01:48:45.924614+0000","last_clean_scrub_stamp":"2023-03-25T01:48:45.924614+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T02:55:58.247062+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.4","version":"20'5","reported_seq":15,"reported_epoch":20,"state":"active+clean","last_fresh":"2023-03-25T01:48:58.920001+0000","last_change":"2023-03-25T01:48:57.404723+0000","last_active":"2023-03-25T01:48:58.920001+0000","last_peered":"2023-03-25T01:48:58.920001+0000","last_clean":"2023-03-25T01:48:58.920001+0000","last_became_active":"2023-03-25T01:48:57.404409+0000","last_became_peered":"2023-03-25T01:48:57.404409+0000","last_unstale":"2023-03-25T01:48:58.920001+0000","last_undegraded":"2023-03-25T01:48:58.920001+0000","last_fullsized":"2023-03-25T01:48:58.920001+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T01:48:55.982031+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T01:48:55.982031+0000","last_clean_scrub_stamp":"2023-03-25T01:48:55.982031+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T07:51:55.253730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":640000,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5,"num_write_kb":627,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"20'88","reported_seq":98,"reported_epoch":20,"state":"active+clean","last_fresh":"2023-03-25T01:49:18.814701+0000","last_change":"2023-03-25T01:48:56.988823+0000","last_active":"2023-03-25T01:49:18.814701+0000","last_peered":"2023-03-25T01:49:18.814701+0000","last_clean":"2023-03-25T01:49:18.814701+0000","last_became_active":"2023-03-25T01:48:56.988568+0000","last_became_peered":"2023-03-25T01:48:56.988568+0000","last_unstale":"2023-03-25T01:49:18.814701+0000","last_undegraded":"2023-03-25T01:49:18.814701+0000","last_fullsized":"2023-03-25T01:49:18.814701+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T01:48:55.982031+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T01:48:55.982031+0000","last_clean_scrub_stamp":"2023-03-25T01:48:55.982031+0000","objects_scrubbed":0,"log_size":88,"log_dups_size":0,"ondisk_log_size":88,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T10:51:06.108055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1310000,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":88,"num_write_kb":1314,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":14,"reported_epoch":19,"state":"active+clean","last_fresh":"2023-03-25T01:48:56.988499+0000","last_change":"2023-03-25T01:48:47.347152+0000","last_active":"2023-03-25T01:48:56.988499+0000","last_peered":"2023-03-25T01:48:56.988499+0000","last_clean":"2023-03-25T01:48:56.988499+0000","last_became_active":"2023-03-25T01:48:47.346919+0000","last_became_peered":"2023-03-25T01:48:47.346919+0000","last_unstale":"2023-03-25T01:48:56.988499+0000","last_undegraded":"2023-03-25T01:48:56.988499+0000","last_fullsized":"2023-03-25T01:48:56.988499+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T01:48:45.924614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T01:48:45.924614+0000","last_clean_scrub_stamp":"2023-03-25T01:48:45.924614+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T02:55:58.247062+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":14,"reported_epoch":19,"state":"active+clean","last_fresh":"2023-03-25T01:48:56.984528+0000","last_change":"2023-03-25T01:48:47.347110+0000","last_active":"2023-03-25T01:48:56.984528+0000","last_peered":"2023-03-25T01:48:56.984528+0000","last_clean":"2023-03-25T01:48:56.984528+0000","last_became_active":"2023-03-25T01:48:47.346832+0000","last_became_peered":"2023-03-25T01:48:47.346832+0000","last_unstale":"2023-03-25T01:48:56.984528+0000","last_undegraded":"2023-03-25T01:48:56.984528+0000","last_fullsized":"2023-03-25T01:48:56.984528+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T01:48:45.924614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T01:48:45.924614+0000","last_clean_scrub_stamp":"2023-03-25T01:48:45.924614+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:34:02.770586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"20'126","reported_seq":136,"reported_epoch":20,"state":"active+clean","last_fresh":"2023-03-25T01:49:17.626037+0000","last_change":"2023-03-25T01:48:56.990341+0000","last_active":"2023-03-25T01:49:17.626037+0000","last_peered":"2023-03-25T01:49:17.626037+0000","last_clean":"2023-03-25T01:49:17.626037+0000","last_became_active":"2023-03-25T01:48:56.990039+0000","last_became_peered":"2023-03-25T01:48:56.990039+0000","last_unstale":"2023-03-25T01:49:17.626037+0000","last_undegraded":"2023-03-25T01:49:17.626037+0000","last_fullsized":"2023-03-25T01:49:17.626037+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T01:48:55.982031+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T01:48:55.982031+0000","last_clean_scrub_stamp":"2023-03-25T01:48:55.982031+0000","objects_scrubbed":0,"log_size":126,"log_dups_size":0,"ondisk_log_size":126,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T06:14:10.792416+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1630000,"num_objects":5,"num_object_clones":0,"num_object_copies":10,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":126,"num_write_kb":1653,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":5,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":3,"num_pg":12,"stat_sum":{"num_bytes":6510000,"num_objects":20,"num_object_clones":0,"num_object_copies":40,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":20,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":403,"num_write_kb":6544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":19,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":13123584,"data_stored":13020000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":403,"ondisk_log_size":403,"up":24,"acting":24,"num_store_stats":6},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":137,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":97,"ondisk_log_size":97,"up":2,"acting":2,"num_store_stats":2},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":6}],"osd_stats":[{"osd":5,"up_from":11,"seq":47244640265,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":104857600,"kb_used":6608,"kb_used_data":1480,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":104850992,"statfs":{"total":107374182400,"available":107367415808,"internally_reserved":0,"allocated":1515520,"data_stored":1366284,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,3,4],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34999999999999998}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.307}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29399999999999998}]},{"osd":3,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.434}]},{"osd":4,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.36499999999999999}]}]},{"osd":4,"up_from":11,"seq":47244640267,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":104857600,"kb_used":8024,"kb_used_data":2768,"kb_used_omap":0,"kb_used_meta":5248,"kb_avail":104849576,"statfs":{"total":107374182400,"available":107365965824,"internally_reserved":0,"allocated":2834432,"data_stored":2676284,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5373952},"hb_peers":[0,1,2,3,5],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.31900000000000001}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.24099999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.25800000000000001}]},{"osd":3,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.41499999999999998}]},{"osd":5,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.435}]}]},{"osd":3,"up_from":11,"seq":47244640266,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":104857600,"kb_used":8736,"kb_used_data":3416,"kb_used_omap":0,"kb_used_meta":5312,"kb_avail":104848864,"statfs":{"total":107374182400,"available":107365236736,"internally_reserved":0,"allocated":3497984,"data_stored":3336284,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5439488},"hb_peers":[0,1,2,4,5],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.30199999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.35599999999999998}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.38400000000000001}]},{"osd":4,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.312}]},{"osd":5,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.32000000000000001}]}]},{"osd":2,"up_from":11,"seq":47244640266,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":104857600,"kb_used":6916,"kb_used_data":1788,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":104850684,"statfs":{"total":107374182400,"available":107367100416,"internally_reserved":0,"allocated":1830912,"data_stored":1676284,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,3,4,5],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.498}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.45200000000000001}]},{"osd":3,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.52800000000000002}]},{"osd":4,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.32100000000000001}]},{"osd":5,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.58299999999999996}]}]},{"osd":0,"up_from":11,"seq":47244640265,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":104857600,"kb_used":8348,"kb_used_data":2708,"kb_used_omap":0,"kb_used_meta":5632,"kb_avail":104849252,"statfs":{"total":107374182400,"available":107365634048,"internally_reserved":0,"allocated":2772992,"data_stored":2616652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5767168},"hb_peers":[1,2,3,4,5],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.32800000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26000000000000001}]},{"osd":3,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.30399999999999999}]},{"osd":4,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.24399999999999999}]},{"osd":5,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":1,"up_from":11,"seq":47244640266,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":104857600,"kb_used":8656,"kb_used_data":3016,"kb_used_omap":0,"kb_used_meta":5632,"kb_avail":104848944,"statfs":{"total":107374182400,"available":107365318656,"internally_reserved":0,"allocated":3088384,"data_stored":2926652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5767168},"hb_peers":[0,2,3,4,5],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.27300000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34799999999999998}]},{"osd":3,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.24399999999999999}]},{"osd":4,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.36499999999999999}]},{"osd":5,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.49099999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1974272,"data_stored":1960000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2289664,"data_stored":2270000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1626112,"data_stored":1610000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":3293184,"data_stored":3270000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2629632,"data_stored":2610000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1310720,"data_stored":1300000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T02:15:44.816 INFO:tasks.ceph:Checking cluster log for badness...
, 2023-03-25T02:15:44.817 DEBUG:teuthology.orchestra.run.smithi087:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/ceph.log | egrep -v 'overall HEALTH_' | egrep -v '\(OSDMAP_FLAGS\)' | egrep -v '\(OSD_' | egrep -v '\(PG_' | egrep -v '\(TOO_FEW_PGS\)' | egrep -v '\(POOL_APP_NOT_ENABLED\)' | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1
, 2023-03-25T02:15:44.885 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/ceph-0 on ubuntu@smithi087.front.sepia.ceph.com

Ignoring malparsed backtrace: 2023-03-25T02:15:48.639 DEBUG:teuthology.orchestra.run.smithi087:> sudo tar cz -f - -C /home/ubuntu/cephtest/archive -- .
, 2023-03-25T02:15:57.238 INFO:teuthology.task.internal:Transferring binaries for coredumps...
, 2023-03-25T02:15:57.288 INFO:teuthology.task.internal: core looks like: /home/teuthworker/archive/benhanokh-2023-03-16_20:57:19-rados-WIP_GBH_snap_mapper_5-distro-default-smithi/7210931/remote/smithi087/coredump/1679708966.134597.core: ELF 64-bit LSB core file x86-64, version 1 (SYSV), SVR4-style, from 'ceph-objectstore-tool --data-path /var/lib/ceph/osd/ceph-3 --journal-path /var/', real uid: 0, effective uid: 0, real gid: 0, effective gid: 0, execfn: '/bin/ceph-objectstore-tool', platform: 'x86_64'

Ignoring malparsed backtrace:  ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
,  1: /lib64/libpthread.so.0(+0x12cf0) [0x7f2181c7ecf0]
,  2: __pthread_mutex_lock()

Ignoring malparsed backtrace: 2023-03-25T02:01:50.041 INFO:teuthology.orchestra.run.smithi087.stdout:{"pg_ready":true,"pg_map":{"version":408,"stamp":"2023-03-25T02:01:48.868189+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":7100368,"num_objects":22,"num_object_clones":0,"num_object_copies":44,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":540,"num_write_kb":8438,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":19,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":500,"ondisk_log_size":500,"up":42,"acting":42,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":42,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":629145600,"kb_used":47288,"kb_used_data":15176,"kb_used_omap":0,"kb_used_meta":32064,"kb_avail":629098312,"statfs":{"total":644245094400,"available":644196671488,"internally_reserved":0,"allocated":15540224,"data_stored":14598440,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":32833536},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002186"},"pg_stats":[{"pgid":"3.b","version":"20'26","reported_seq":36,"reported_epoch":20,"state":"active+clean","last_fresh":"2023-03-25T01:49:07.722753+0000","last_change":"2023-03-25T01:48:56.990468+0000","last_active":"2023-03-25T01:49:07.722753+0000","last_peered":"2023-03-25T01:49:07.722753+0000","last_clean":"2023-03-25T01:49:07.722753+0000","last_became_active":"2023-03-25T01:48:56.990194+0000","last_became_peered":"2023-03-25T01:48:56.990194+0000","last_unstale":"2023-03-25T01:49:07.722753+0000","last_undegraded":"2023-03-25T01:49:07.722753+0000","last_fullsized":"2023-03-25T01:49:07.722753+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T01:48:55.982031+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T01:48:55.982031+0000","last_clean_scrub_stamp":"2023-03-25T01:48:55.982031+0000","objects_scrubbed":0,"log_size":26,"log_dups_size":0,"ondisk_log_size":26,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T07:51:55.253730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":330000,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":26,"num_write_kb":335,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":10,"reported_epoch":19,"state":"active+clean","last_fresh":"2023-03-25T01:48:58.910560+0000","last_change":"2023-03-25T01:48:56.990354+0000","last_active":"2023-03-25T01:48:58.910560+0000","last_peered":"2023-03-25T01:48:58.910560+0000","last_clean":"2023-03-25T01:48:58.910560+0000","last_became_active":"2023-03-25T01:48:56.989754+0000","last_became_peered":"2023-03-25T01:48:56.989754+0000","last_unstale":"2023-03-25T01:48:58.910560+0000","last_undegraded":"2023-03-25T01:48:58.910560+0000","last_fullsized":"2023-03-25T01:48:58.910560+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T01:48:55.982031+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T01:48:55.982031+0000","last_clean_scrub_stamp":"2023-03-25T01:48:55.982031+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:18:38.285450+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"20'64","reported_seq":74,"reported_epoch":20,"state":"active+clean","last_fresh":"2023-03-25T01:49:12.193662+0000","last_change":"2023-03-25T01:48:57.404560+0000","last_active":"2023-03-25T01:49:12.193662+0000","last_peered":"2023-03-25T01:49:12.193662+0000","last_clean":"2023-03-25T01:49:12.193662+0000","last_became_active":"2023-03-25T01:48:57.404333+0000","last_became_peered":"2023-03-25T01:48:57.404333+0000","last_unstale":"2023-03-25T01:49:12.193662+0000","last_undegraded":"2023-03-25T01:49:12.193662+0000","last_fullsized":"2023-03-25T01:49:12.193662+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T01:48:55.982031+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T01:48:55.982031+0000","last_clean_scrub_stamp":"2023-03-25T01:48:55.982031+0000","objects_scrubbed":0,"log_size":64,"log_dups_size":0,"ondisk_log_size":64,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:18:38.285450+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":970000,"num_objects":3,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":64,"num_write_kb":978,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":3,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2],"acting":[4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":10,"reported_epoch":19,"state":"active+clean","last_fresh":"2023-03-25T01:48:58.909182+0000","last_change":"2023-03-25T01:48:56.989694+0000","last_active":"2023-03-25T01:48:58.909182+0000","last_peered":"2023-03-25T01:48:58.909182+0000","last_clean":"2023-03-25T01:48:58.909182+0000","last_became_active":"2023-03-25T01:48:56.989461+0000","last_became_peered":"2023-03-25T01:48:56.989461+0000","last_unstale":"2023-03-25T01:48:58.909182+0000","last_undegraded":"2023-03-25T01:48:58.909182+0000","last_fullsized":"2023-03-25T01:48:58.909182+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T01:48:55.982031+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T01:48:55.982031+0000","last_clean_scrub_stamp":"2023-03-25T01:48:55.982031+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T12:21:35.179737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":14,"reported_epoch":19,"state":"active+clean","last_fresh":"2023-03-25T01:48:56.988435+0000","last_change":"2023-03-25T01:48:47.347051+0000","last_active":"2023-03-25T01:48:56.988435+0000","last_peered":"2023-03-25T01:48:56.988435+0000","last_clean":"2023-03-25T01:48:56.988435+0000","last_became_active":"2023-03-25T01:48:47.346812+0000","last_became_peered":"2023-03-25T01:48:47.346812+0000","last_unstale":"2023-03-25T01:48:56.988435+0000","last_undegraded":"2023-03-25T01:48:56.988435+0000","last_fullsized":"2023-03-25T01:48:56.988435+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T01:48:45.924614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T01:48:45.924614+0000","last_clean_scrub_stamp":"2023-03-25T01:48:45.924614+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T07:21:10.392042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"20'30","reported_seq":40,"reported_epoch":20,"state":"active+clean","last_fresh":"2023-03-25T01:49:04.130697+0000","last_change":"2023-03-25T01:48:56.990522+0000","last_active":"2023-03-25T01:49:04.130697+0000","last_peered":"2023-03-25T01:49:04.130697+0000","last_clean":"2023-03-25T01:49:04.130697+0000","last_became_active":"2023-03-25T01:48:56.990276+0000","last_became_peered":"2023-03-25T01:48:56.990276+0000","last_unstale":"2023-03-25T01:49:04.130697+0000","last_undegraded":"2023-03-25T01:49:04.130697+0000","last_fullsized":"2023-03-25T01:49:04.130697+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T01:48:55.982031+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T01:48:55.982031+0000","last_clean_scrub_stamp":"2023-03-25T01:48:55.982031+0000","objects_scrubbed":0,"log_size":30,"log_dups_size":0,"ondisk_log_size":30,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:43:07.120795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":650000,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":30,"num_write_kb":649,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":15,"reported_epoch":19,"state":"active+clean","last_fresh":"2023-03-25T01:48:56.988086+0000","last_change":"2023-03-25T01:48:46.941510+0000","last_active":"2023-03-25T01:48:56.988086+0000","last_peered":"2023-03-25T01:48:56.988086+0000","last_clean":"2023-03-25T01:48:56.988086+0000","last_became_active":"2023-03-25T01:48:46.941266+0000","last_became_peered":"2023-03-25T01:48:46.941266+0000","last_unstale":"2023-03-25T01:48:56.988086+0000","last_undegraded":"2023-03-25T01:48:56.988086+0000","last_fullsized":"2023-03-25T01:48:56.988086+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T01:48:45.924614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T01:48:45.924614+0000","last_clean_scrub_stamp":"2023-03-25T01:48:45.924614+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:34:02.770586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1],"acting":[5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":10,"reported_epoch":19,"state":"active+clean","last_fresh":"2023-03-25T01:48:58.909220+0000","last_change":"2023-03-25T01:48:56.990419+0000","last_active":"2023-03-25T01:48:58.909220+0000","last_peered":"2023-03-25T01:48:58.909220+0000","last_clean":"2023-03-25T01:48:58.909220+0000","last_became_active":"2023-03-25T01:48:56.990188+0000","last_became_peered":"2023-03-25T01:48:56.990188+0000","last_unstale":"2023-03-25T01:48:58.909220+0000","last_undegraded":"2023-03-25T01:48:58.909220+0000","last_fullsized":"2023-03-25T01:48:58.909220+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T01:48:55.982031+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T01:48:55.982031+0000","last_clean_scrub_stamp":"2023-03-25T01:48:55.982031+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:43:07.120795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":14,"reported_epoch":19,"state":"active+clean","last_fresh":"2023-03-25T01:48:57.402752+0000","last_change":"2023-03-25T01:48:46.942191+0000","last_active":"2023-03-25T01:48:57.402752+0000","last_peered":"2023-03-25T01:48:57.402752+0000","last_clean":"2023-03-25T01:48:57.402752+0000","last_became_active":"2023-03-25T01:48:46.941918+0000","last_became_peered":"2023-03-25T01:48:46.941918+0000","last_unstale":"2023-03-25T01:48:57.402752+0000","last_undegraded":"2023-03-25T01:48:57.402752+0000","last_fullsized":"2023-03-25T01:48:57.402752+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T01:48:45.924614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T01:48:45.924614+0000","last_clean_scrub_stamp":"2023-03-25T01:48:45.924614+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T10:36:32.544700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5],"acting":[2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":10,"reported_epoch":20,"state":"active+clean","last_fresh":"2023-03-25T01:48:57.985996+0000","last_change":"2023-03-25T01:48:57.404656+0000","last_active":"2023-03-25T01:48:57.985996+0000","last_peered":"2023-03-25T01:48:57.985996+0000","last_clean":"2023-03-25T01:48:57.985996+0000","last_became_active":"2023-03-25T01:48:57.404416+0000","last_became_peered":"2023-03-25T01:48:57.404416+0000","last_unstale":"2023-03-25T01:48:57.985996+0000","last_undegraded":"2023-03-25T01:48:57.985996+0000","last_fullsized":"2023-03-25T01:48:57.985996+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T01:48:55.982031+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T01:48:55.982031+0000","last_clean_scrub_stamp":"2023-03-25T01:48:55.982031+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T12:21:35.179737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":15,"reported_epoch":19,"state":"active+clean","last_fresh":"2023-03-25T01:48:56.987983+0000","last_change":"2023-03-25T01:48:47.283824+0000","last_active":"2023-03-25T01:48:56.987983+0000","last_peered":"2023-03-25T01:48:56.987983+0000","last_clean":"2023-03-25T01:48:56.987983+0000","last_became_active":"2023-03-25T01:48:47.283534+0000","last_became_peered":"2023-03-25T01:48:47.283534+0000","last_unstale":"2023-03-25T01:48:56.987983+0000","last_undegraded":"2023-03-25T01:48:56.987983+0000","last_fullsized":"2023-03-25T01:48:56.987983+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T01:48:45.924614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T01:48:45.924614+0000","last_clean_scrub_stamp":"2023-03-25T01:48:45.924614+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T10:36:32.544700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4],"acting":[5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"13'97","reported_seq":142,"reported_epoch":19,"state":"active+clean","last_fresh":"2023-03-25T01:48:56.988118+0000","last_change":"2023-03-25T01:48:45.934080+0000","last_active":"2023-03-25T01:48:56.988118+0000","last_peered":"2023-03-25T01:48:56.988118+0000","last_clean":"2023-03-25T01:48:56.988118+0000","last_became_active":"2023-03-25T01:48:45.933746+0000","last_became_peered":"2023-03-25T01:48:45.933746+0000","last_unstale":"2023-03-25T01:48:56.988118+0000","last_undegraded":"2023-03-25T01:48:56.988118+0000","last_fullsized":"2023-03-25T01:48:56.988118+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T01:48:44.920729+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T01:48:44.920729+0000","last_clean_scrub_stamp":"2023-03-25T01:48:44.920729+0000","objects_scrubbed":0,"log_size":97,"log_dups_size":0,"ondisk_log_size":97,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:43:52.243010+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":137,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.2","version":"20'30","reported_seq":40,"reported_epoch":20,"state":"active+clean","last_fresh":"2023-03-25T01:49:06.431894+0000","last_change":"2023-03-25T01:48:56.991149+0000","last_active":"2023-03-25T01:49:06.431894+0000","last_peered":"2023-03-25T01:49:06.431894+0000","last_clean":"2023-03-25T01:49:06.431894+0000","last_became_active":"2023-03-25T01:48:56.990889+0000","last_became_peered":"2023-03-25T01:48:56.990889+0000","last_unstale":"2023-03-25T01:49:06.431894+0000","last_undegraded":"2023-03-25T01:49:06.431894+0000","last_fullsized":"2023-03-25T01:49:06.431894+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T01:48:55.982031+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T01:48:55.982031+0000","last_clean_scrub_stamp":"2023-03-25T01:48:55.982031+0000","objects_scrubbed":0,"log_size":30,"log_dups_size":0,"ondisk_log_size":30,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T10:51:06.108055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":650000,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":30,"num_write_kb":649,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5],"acting":[3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":14,"reported_epoch":19,"state":"active+clean","last_fresh":"2023-03-25T01:48:56.984507+0000","last_change":"2023-03-25T01:48:46.943089+0000","last_active":"2023-03-25T01:48:56.984507+0000","last_peered":"2023-03-25T01:48:56.984507+0000","last_clean":"2023-03-25T01:48:56.984507+0000","last_became_active":"2023-03-25T01:48:46.942843+0000","last_became_peered":"2023-03-25T01:48:46.942843+0000","last_unstale":"2023-03-25T01:48:56.984507+0000","last_undegraded":"2023-03-25T01:48:56.984507+0000","last_fullsized":"2023-03-25T01:48:56.984507+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T01:48:45.924614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T01:48:45.924614+0000","last_clean_scrub_stamp":"2023-03-25T01:48:45.924614+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T07:21:10.392042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5],"acting":[3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1","version":"20'34","reported_seq":44,"reported_epoch":20,"state":"active+clean","last_fresh":"2023-03-25T01:49:13.951565+0000","last_change":"2023-03-25T01:48:56.989334+0000","last_active":"2023-03-25T01:49:13.951565+0000","last_peered":"2023-03-25T01:49:13.951565+0000","last_clean":"2023-03-25T01:49:13.951565+0000","last_became_active":"2023-03-25T01:48:56.989079+0000","last_became_peered":"2023-03-25T01:48:56.989079+0000","last_unstale":"2023-03-25T01:49:13.951565+0000","last_undegraded":"2023-03-25T01:49:13.951565+0000","last_fullsized":"2023-03-25T01:49:13.951565+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T01:48:55.982031+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T01:48:55.982031+0000","last_clean_scrub_stamp":"2023-03-25T01:48:55.982031+0000","objects_scrubbed":0,"log_size":34,"log_dups_size":0,"ondisk_log_size":34,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T07:51:55.253730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":330000,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":34,"num_write_kb":339,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":14,"reported_epoch":19,"state":"active+clean","last_fresh":"2023-03-25T01:48:56.984518+0000","last_change":"2023-03-25T01:48:47.283615+0000","last_active":"2023-03-25T01:48:56.984518+0000","last_peered":"2023-03-25T01:48:56.984518+0000","last_clean":"2023-03-25T01:48:56.984518+0000","last_became_active":"2023-03-25T01:48:47.283359+0000","last_became_peered":"2023-03-25T01:48:47.283359+0000","last_unstale":"2023-03-25T01:48:56.984518+0000","last_undegraded":"2023-03-25T01:48:56.984518+0000","last_fullsized":"2023-03-25T01:48:56.984518+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T01:48:45.924614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T01:48:45.924614+0000","last_clean_scrub_stamp":"2023-03-25T01:48:45.924614+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T02:55:58.247062+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.4","version":"20'5","reported_seq":15,"reported_epoch":20,"state":"active+clean","last_fresh":"2023-03-25T01:48:58.920001+0000","last_change":"2023-03-25T01:48:57.404723+0000","last_active":"2023-03-25T01:48:58.920001+0000","last_peered":"2023-03-25T01:48:58.920001+0000","last_clean":"2023-03-25T01:48:58.920001+0000","last_became_active":"2023-03-25T01:48:57.404409+0000","last_became_peered":"2023-03-25T01:48:57.404409+0000","last_unstale":"2023-03-25T01:48:58.920001+0000","last_undegraded":"2023-03-25T01:48:58.920001+0000","last_fullsized":"2023-03-25T01:48:58.920001+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T01:48:55.982031+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T01:48:55.982031+0000","last_clean_scrub_stamp":"2023-03-25T01:48:55.982031+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T07:51:55.253730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":640000,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5,"num_write_kb":627,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"20'88","reported_seq":98,"reported_epoch":20,"state":"active+clean","last_fresh":"2023-03-25T01:49:18.814701+0000","last_change":"2023-03-25T01:48:56.988823+0000","last_active":"2023-03-25T01:49:18.814701+0000","last_peered":"2023-03-25T01:49:18.814701+0000","last_clean":"2023-03-25T01:49:18.814701+0000","last_became_active":"2023-03-25T01:48:56.988568+0000","last_became_peered":"2023-03-25T01:48:56.988568+0000","last_unstale":"2023-03-25T01:49:18.814701+0000","last_undegraded":"2023-03-25T01:49:18.814701+0000","last_fullsized":"2023-03-25T01:49:18.814701+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T01:48:55.982031+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T01:48:55.982031+0000","last_clean_scrub_stamp":"2023-03-25T01:48:55.982031+0000","objects_scrubbed":0,"log_size":88,"log_dups_size":0,"ondisk_log_size":88,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T10:51:06.108055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1310000,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":88,"num_write_kb":1314,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":14,"reported_epoch":19,"state":"active+clean","last_fresh":"2023-03-25T01:48:56.988499+0000","last_change":"2023-03-25T01:48:47.347152+0000","last_active":"2023-03-25T01:48:56.988499+0000","last_peered":"2023-03-25T01:48:56.988499+0000","last_clean":"2023-03-25T01:48:56.988499+0000","last_became_active":"2023-03-25T01:48:47.346919+0000","last_became_peered":"2023-03-25T01:48:47.346919+0000","last_unstale":"2023-03-25T01:48:56.988499+0000","last_undegraded":"2023-03-25T01:48:56.988499+0000","last_fullsized":"2023-03-25T01:48:56.988499+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T01:48:45.924614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T01:48:45.924614+0000","last_clean_scrub_stamp":"2023-03-25T01:48:45.924614+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T02:55:58.247062+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":14,"reported_epoch":19,"state":"active+clean","last_fresh":"2023-03-25T01:48:56.984528+0000","last_change":"2023-03-25T01:48:47.347110+0000","last_active":"2023-03-25T01:48:56.984528+0000","last_peered":"2023-03-25T01:48:56.984528+0000","last_clean":"2023-03-25T01:48:56.984528+0000","last_became_active":"2023-03-25T01:48:47.346832+0000","last_became_peered":"2023-03-25T01:48:47.346832+0000","last_unstale":"2023-03-25T01:48:56.984528+0000","last_undegraded":"2023-03-25T01:48:56.984528+0000","last_fullsized":"2023-03-25T01:48:56.984528+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T01:48:45.924614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T01:48:45.924614+0000","last_clean_scrub_stamp":"2023-03-25T01:48:45.924614+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:34:02.770586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"20'126","reported_seq":136,"reported_epoch":20,"state":"active+clean","last_fresh":"2023-03-25T01:49:17.626037+0000","last_change":"2023-03-25T01:48:56.990341+0000","last_active":"2023-03-25T01:49:17.626037+0000","last_peered":"2023-03-25T01:49:17.626037+0000","last_clean":"2023-03-25T01:49:17.626037+0000","last_became_active":"2023-03-25T01:48:56.990039+0000","last_became_peered":"2023-03-25T01:48:56.990039+0000","last_unstale":"2023-03-25T01:49:17.626037+0000","last_undegraded":"2023-03-25T01:49:17.626037+0000","last_fullsized":"2023-03-25T01:49:17.626037+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T01:48:55.982031+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T01:48:55.982031+0000","last_clean_scrub_stamp":"2023-03-25T01:48:55.982031+0000","objects_scrubbed":0,"log_size":126,"log_dups_size":0,"ondisk_log_size":126,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T06:14:10.792416+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1630000,"num_objects":5,"num_object_clones":0,"num_object_copies":10,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":126,"num_write_kb":1653,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":5,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":3,"num_pg":12,"stat_sum":{"num_bytes":6510000,"num_objects":20,"num_object_clones":0,"num_object_copies":40,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":20,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":403,"num_write_kb":6544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":19,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":13123584,"data_stored":13020000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":403,"ondisk_log_size":403,"up":24,"acting":24,"num_store_stats":6},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":137,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":97,"ondisk_log_size":97,"up":2,"acting":2,"num_store_stats":2},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":6}],"osd_stats":[{"osd":5,"up_from":11,"seq":47244640265,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":104857600,"kb_used":6608,"kb_used_data":1480,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":104850992,"statfs":{"total":107374182400,"available":107367415808,"internally_reserved":0,"allocated":1515520,"data_stored":1366284,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,3,4],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34999999999999998}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.307}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29399999999999998}]},{"osd":3,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.434}]},{"osd":4,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.36499999999999999}]}]},{"osd":4,"up_from":11,"seq":47244640267,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":104857600,"kb_used":8024,"kb_used_data":2768,"kb_used_omap":0,"kb_used_meta":5248,"kb_avail":104849576,"statfs":{"total":107374182400,"available":107365965824,"internally_reserved":0,"allocated":2834432,"data_stored":2676284,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5373952},"hb_peers":[0,1,2,3,5],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.31900000000000001}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.24099999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.25800000000000001}]},{"osd":3,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.41499999999999998}]},{"osd":5,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.435}]}]},{"osd":3,"up_from":11,"seq":47244640266,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":104857600,"kb_used":8736,"kb_used_data":3416,"kb_used_omap":0,"kb_used_meta":5312,"kb_avail":104848864,"statfs":{"total":107374182400,"available":107365236736,"internally_reserved":0,"allocated":3497984,"data_stored":3336284,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5439488},"hb_peers":[0,1,2,4,5],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.30199999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.35599999999999998}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.38400000000000001}]},{"osd":4,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.312}]},{"osd":5,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.32000000000000001}]}]},{"osd":2,"up_from":11,"seq":47244640266,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":104857600,"kb_used":6916,"kb_used_data":1788,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":104850684,"statfs":{"total":107374182400,"available":107367100416,"internally_reserved":0,"allocated":1830912,"data_stored":1676284,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,3,4,5],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.498}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.45200000000000001}]},{"osd":3,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.52800000000000002}]},{"osd":4,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.32100000000000001}]},{"osd":5,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.58299999999999996}]}]},{"osd":0,"up_from":11,"seq":47244640265,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":104857600,"kb_used":8348,"kb_used_data":2708,"kb_used_omap":0,"kb_used_meta":5632,"kb_avail":104849252,"statfs":{"total":107374182400,"available":107365634048,"internally_reserved":0,"allocated":2772992,"data_stored":2616652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5767168},"hb_peers":[1,2,3,4,5],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.32800000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26000000000000001}]},{"osd":3,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.30399999999999999}]},{"osd":4,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.24399999999999999}]},{"osd":5,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":1,"up_from":11,"seq":47244640266,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":104857600,"kb_used":8656,"kb_used_data":3016,"kb_used_omap":0,"kb_used_meta":5632,"kb_avail":104848944,"statfs":{"total":107374182400,"available":107365318656,"internally_reserved":0,"allocated":3088384,"data_stored":2926652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5767168},"hb_peers":[0,2,3,4,5],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.27300000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34799999999999998}]},{"osd":3,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.24399999999999999}]},{"osd":4,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.36499999999999999}]},{"osd":5,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.49099999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1974272,"data_stored":1960000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2289664,"data_stored":2270000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1626112,"data_stored":1610000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":3293184,"data_stored":3270000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2629632,"data_stored":2610000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1310720,"data_stored":1300000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T02:01:50.044 INFO:tasks.ceph:pgid 3.b last_scrub_stamp 2023-03-25T01:48:55.982031+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=48, tm_sec=55, tm_wday=5, tm_yday=84, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=49, tm_sec=28, tm_wday=5, tm_yday=84, tm_isdst=0)
, 2023-03-25T02:01:50.044 INFO:tasks.ceph:pgid 3.a last_scrub_stamp 2023-03-25T01:48:55.982031+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=48, tm_sec=55, tm_wday=5, tm_yday=84, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=49, tm_sec=28, tm_wday=5, tm_yday=84, tm_isdst=0)

Ignoring malparsed backtrace: 2023-03-25T02:03:11.095 DEBUG:teuthology.orchestra.run.smithi087:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T02:03:11.419 INFO:teuthology.orchestra.run.smithi087.stdout:
, dumped all

Ignoring malparsed backtrace: 2023-03-25T02:03:58.964 INFO:tasks.ceph:pgid 3.8 last_scrub_stamp 2023-03-25T01:48:55.982031+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=48, tm_sec=55, tm_wday=5, tm_yday=84, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=49, tm_sec=28, tm_wday=5, tm_yday=84, tm_isdst=0)
, 2023-03-25T02:03:58.964 INFO:tasks.ceph:pgid 2.4 last_scrub_stamp 2023-03-25T01:48:45.924614+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=48, tm_sec=45, tm_wday=5, tm_yday=84, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=49, tm_sec=28, tm_wday=5, tm_yday=84, tm_isdst=0)
, 2023-03-25T02:03:58.964 INFO:tasks.ceph:pgid 3.5 last_scrub_stamp 2023-03-25T01:48:55.982031+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=48, tm_sec=55, tm_wday=5, tm_yday=84, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=49, tm_sec=28, tm_wday=5, tm_yday=84, tm_isdst=0)

Ignoring malparsed backtrace: 2023-03-25T02:05:20.356 INFO:tasks.ceph:pgid 3.b last_scrub_stamp 2023-03-25T01:48:55.982031+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=48, tm_sec=55, tm_wday=5, tm_yday=84, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=49, tm_sec=28, tm_wday=5, tm_yday=84, tm_isdst=0)
, 2023-03-25T02:05:20.356 INFO:tasks.ceph:pgid 3.a last_scrub_stamp 2023-03-25T01:48:55.982031+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=48, tm_sec=55, tm_wday=5, tm_yday=84, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=49, tm_sec=28, tm_wday=5, tm_yday=84, tm_isdst=0)
, 2023-03-25T02:05:20.356 INFO:tasks.ceph:pgid 3.9 last_scrub_stamp 2023-03-25T01:48:55.982031+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=48, tm_sec=55, tm_wday=5, tm_yday=84, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=49, tm_sec=28, tm_wday=5, tm_yday=84, tm_isdst=0)

Ignoring malparsed backtrace: 2023-03-25T02:06:07.907 INFO:tasks.ceph:pgid 3.3 last_scrub_stamp 2023-03-25T01:48:55.982031+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=48, tm_sec=55, tm_wday=5, tm_yday=84, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=49, tm_sec=28, tm_wday=5, tm_yday=84, tm_isdst=0)
, 2023-03-25T02:06:07.907 INFO:tasks.ceph:pgid 2.1 last_scrub_stamp 2023-03-25T01:48:45.924614+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=48, tm_sec=45, tm_wday=5, tm_yday=84, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=49, tm_sec=28, tm_wday=5, tm_yday=84, tm_isdst=0)
, 2023-03-25T02:06:07.907 INFO:tasks.ceph:pgid 3.0 last_scrub_stamp 2023-03-25T01:48:55.982031+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=48, tm_sec=55, tm_wday=5, tm_yday=84, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=49, tm_sec=28, tm_wday=5, tm_yday=84, tm_isdst=0)

Ignoring malparsed backtrace: 2023-03-25T02:07:29.309 INFO:tasks.ceph:pgid 2.4 last_scrub_stamp 2023-03-25T01:48:45.924614+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=48, tm_sec=45, tm_wday=5, tm_yday=84, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=49, tm_sec=28, tm_wday=5, tm_yday=84, tm_isdst=0)
, 2023-03-25T02:07:29.309 INFO:tasks.ceph:pgid 3.5 last_scrub_stamp 2023-03-25T01:48:55.982031+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=48, tm_sec=55, tm_wday=5, tm_yday=84, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=49, tm_sec=28, tm_wday=5, tm_yday=84, tm_isdst=0)
, 2023-03-25T02:07:29.309 INFO:tasks.ceph:pgid 2.2 last_scrub_stamp 2023-03-25T01:48:45.924614+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=48, tm_sec=45, tm_wday=5, tm_yday=84, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=49, tm_sec=28, tm_wday=5, tm_yday=84, tm_isdst=0)

Ignoring malparsed backtrace: 2023-03-25T02:08:16.851 INFO:tasks.ceph:pgid 1.0 last_scrub_stamp 2023-03-25T01:48:44.920729+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=48, tm_sec=44, tm_wday=5, tm_yday=84, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=49, tm_sec=28, tm_wday=5, tm_yday=84, tm_isdst=0)
, 2023-03-25T02:08:16.851 INFO:tasks.ceph:pgid 3.2 last_scrub_stamp 2023-03-25T01:48:55.982031+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=48, tm_sec=55, tm_wday=5, tm_yday=84, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=49, tm_sec=28, tm_wday=5, tm_yday=84, tm_isdst=0)
, 2023-03-25T02:08:16.852 INFO:tasks.ceph:pgid 2.0 last_scrub_stamp 2023-03-25T01:48:45.924614+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=48, tm_sec=45, tm_wday=5, tm_yday=84, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=49, tm_sec=28, tm_wday=5, tm_yday=84, tm_isdst=0)

Ignoring malparsed backtrace: 2023-03-25T02:09:38.242 INFO:tasks.ceph:pgid 2.1 last_scrub_stamp 2023-03-25T01:48:45.924614+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=48, tm_sec=45, tm_wday=5, tm_yday=84, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=49, tm_sec=28, tm_wday=5, tm_yday=84, tm_isdst=0)
, 2023-03-25T02:09:38.242 INFO:tasks.ceph:pgid 3.0 last_scrub_stamp 2023-03-25T01:48:55.982031+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=48, tm_sec=55, tm_wday=5, tm_yday=84, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=49, tm_sec=28, tm_wday=5, tm_yday=84, tm_isdst=0)
, 2023-03-25T02:09:38.242 INFO:tasks.ceph:pgid 2.3 last_scrub_stamp 2023-03-25T01:48:45.924614+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=48, tm_sec=45, tm_wday=5, tm_yday=84, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=49, tm_sec=28, tm_wday=5, tm_yday=84, tm_isdst=0)

Ignoring malparsed backtrace: 2023-03-25T02:10:25.793 INFO:tasks.ceph:pgid 2.5 last_scrub_stamp 2023-03-25T01:48:45.924614+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=48, tm_sec=45, tm_wday=5, tm_yday=84, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=49, tm_sec=28, tm_wday=5, tm_yday=84, tm_isdst=0)
, 2023-03-25T02:10:25.793 INFO:tasks.ceph:pgid 3.4 last_scrub_stamp 2023-03-25T01:48:55.982031+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=48, tm_sec=55, tm_wday=5, tm_yday=84, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=49, tm_sec=28, tm_wday=5, tm_yday=84, tm_isdst=0)
, 2023-03-25T02:10:25.793 INFO:tasks.ceph:pgid 3.7 last_scrub_stamp 2023-03-25T01:48:55.982031+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=48, tm_sec=55, tm_wday=5, tm_yday=84, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=49, tm_sec=28, tm_wday=5, tm_yday=84, tm_isdst=0)

Ignoring malparsed backtrace: 2023-03-25T02:11:47.190 INFO:tasks.ceph:pgid 3.2 last_scrub_stamp 2023-03-25T01:48:55.982031+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=48, tm_sec=55, tm_wday=5, tm_yday=84, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=49, tm_sec=28, tm_wday=5, tm_yday=84, tm_isdst=0)
, 2023-03-25T02:11:47.190 INFO:tasks.ceph:pgid 2.0 last_scrub_stamp 2023-03-25T01:48:45.924614+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=48, tm_sec=45, tm_wday=5, tm_yday=84, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=49, tm_sec=28, tm_wday=5, tm_yday=84, tm_isdst=0)
, 2023-03-25T02:11:47.190 INFO:tasks.ceph:pgid 3.1 last_scrub_stamp 2023-03-25T01:48:55.982031+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=48, tm_sec=55, tm_wday=5, tm_yday=84, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=49, tm_sec=28, tm_wday=5, tm_yday=84, tm_isdst=0)

Ignoring malparsed backtrace: 2023-03-25T02:12:34.721 INFO:tasks.ceph:pgid 2.7 last_scrub_stamp 2023-03-25T01:48:45.924614+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=48, tm_sec=45, tm_wday=5, tm_yday=84, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=49, tm_sec=28, tm_wday=5, tm_yday=84, tm_isdst=0)
, 2023-03-25T02:12:34.722 INFO:tasks.ceph:pgid 3.6 last_scrub_stamp 2023-03-25T01:48:55.982031+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=48, tm_sec=55, tm_wday=5, tm_yday=84, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=49, tm_sec=28, tm_wday=5, tm_yday=84, tm_isdst=0)
, 2023-03-25T02:12:34.722 INFO:tasks.ceph:Still waiting for all pgs to be scrubbed.

Ignoring malparsed backtrace: 2023-03-25T02:13:56.118 INFO:tasks.ceph:pgid 3.4 last_scrub_stamp 2023-03-25T01:48:55.982031+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=48, tm_sec=55, tm_wday=5, tm_yday=84, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=49, tm_sec=28, tm_wday=5, tm_yday=84, tm_isdst=0)
, 2023-03-25T02:13:56.119 INFO:tasks.ceph:pgid 3.7 last_scrub_stamp 2023-03-25T01:48:55.982031+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=48, tm_sec=55, tm_wday=5, tm_yday=84, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=49, tm_sec=28, tm_wday=5, tm_yday=84, tm_isdst=0)
, 2023-03-25T02:13:56.119 INFO:tasks.ceph:pgid 2.6 last_scrub_stamp 2023-03-25T01:48:45.924614+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=48, tm_sec=45, tm_wday=5, tm_yday=84, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=49, tm_sec=28, tm_wday=5, tm_yday=84, tm_isdst=0)

Ignoring malparsed backtrace: 2023-03-25T02:15:03.973 INFO:teuthology.orchestra.run.smithi087.stdout:
, dumped all
, 2023-03-25T02:15:03.986 INFO:teuthology.orchestra.run.smithi087.stdout:{"pg_ready":true,"pg_map":{"version":805,"stamp":"2023-03-25T02:15:03.017634+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":7100368,"num_objects":22,"num_object_clones":0,"num_object_copies":44,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":540,"num_write_kb":8438,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":19,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":500,"ondisk_log_size":500,"up":42,"acting":42,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":42,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":629145600,"kb_used":47288,"kb_used_data":15176,"kb_used_omap":0,"kb_used_meta":32064,"kb_avail":629098312,"statfs":{"total":644245094400,"available":644196671488,"internally_reserved":0,"allocated":15540224,"data_stored":14598440,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":32833536},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002272"},"pg_stats":[{"pgid":"3.b","version":"20'26","reported_seq":36,"reported_epoch":20,"state":"active+clean","last_fresh":"2023-03-25T01:49:07.722753+0000","last_change":"2023-03-25T01:48:56.990468+0000","last_active":"2023-03-25T01:49:07.722753+0000","last_peered":"2023-03-25T01:49:07.722753+0000","last_clean":"2023-03-25T01:49:07.722753+0000","last_became_active":"2023-03-25T01:48:56.990194+0000","last_became_peered":"2023-03-25T01:48:56.990194+0000","last_unstale":"2023-03-25T01:49:07.722753+0000","last_undegraded":"2023-03-25T01:49:07.722753+0000","last_fullsized":"2023-03-25T01:49:07.722753+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T01:48:55.982031+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T01:48:55.982031+0000","last_clean_scrub_stamp":"2023-03-25T01:48:55.982031+0000","objects_scrubbed":0,"log_size":26,"log_dups_size":0,"ondisk_log_size":26,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T07:51:55.253730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":330000,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":26,"num_write_kb":335,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":10,"reported_epoch":19,"state":"active+clean","last_fresh":"2023-03-25T01:48:58.910560+0000","last_change":"2023-03-25T01:48:56.990354+0000","last_active":"2023-03-25T01:48:58.910560+0000","last_peered":"2023-03-25T01:48:58.910560+0000","last_clean":"2023-03-25T01:48:58.910560+0000","last_became_active":"2023-03-25T01:48:56.989754+0000","last_became_peered":"2023-03-25T01:48:56.989754+0000","last_unstale":"2023-03-25T01:48:58.910560+0000","last_undegraded":"2023-03-25T01:48:58.910560+0000","last_fullsized":"2023-03-25T01:48:58.910560+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T01:48:55.982031+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T01:48:55.982031+0000","last_clean_scrub_stamp":"2023-03-25T01:48:55.982031+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:18:38.285450+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"20'64","reported_seq":74,"reported_epoch":20,"state":"active+clean","last_fresh":"2023-03-25T01:49:12.193662+0000","last_change":"2023-03-25T01:48:57.404560+0000","last_active":"2023-03-25T01:49:12.193662+0000","last_peered":"2023-03-25T01:49:12.193662+0000","last_clean":"2023-03-25T01:49:12.193662+0000","last_became_active":"2023-03-25T01:48:57.404333+0000","last_became_peered":"2023-03-25T01:48:57.404333+0000","last_unstale":"2023-03-25T01:49:12.193662+0000","last_undegraded":"2023-03-25T01:49:12.193662+0000","last_fullsized":"2023-03-25T01:49:12.193662+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T01:48:55.982031+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T01:48:55.982031+0000","last_clean_scrub_stamp":"2023-03-25T01:48:55.982031+0000","objects_scrubbed":0,"log_size":64,"log_dups_size":0,"ondisk_log_size":64,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:18:38.285450+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":970000,"num_objects":3,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":64,"num_write_kb":978,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":3,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2],"acting":[4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":10,"reported_epoch":19,"state":"active+clean","last_fresh":"2023-03-25T01:48:58.909182+0000","last_change":"2023-03-25T01:48:56.989694+0000","last_active":"2023-03-25T01:48:58.909182+0000","last_peered":"2023-03-25T01:48:58.909182+0000","last_clean":"2023-03-25T01:48:58.909182+0000","last_became_active":"2023-03-25T01:48:56.989461+0000","last_became_peered":"2023-03-25T01:48:56.989461+0000","last_unstale":"2023-03-25T01:48:58.909182+0000","last_undegraded":"2023-03-25T01:48:58.909182+0000","last_fullsized":"2023-03-25T01:48:58.909182+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T01:48:55.982031+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T01:48:55.982031+0000","last_clean_scrub_stamp":"2023-03-25T01:48:55.982031+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T12:21:35.179737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":14,"reported_epoch":19,"state":"active+clean","last_fresh":"2023-03-25T01:48:56.988435+0000","last_change":"2023-03-25T01:48:47.347051+0000","last_active":"2023-03-25T01:48:56.988435+0000","last_peered":"2023-03-25T01:48:56.988435+0000","last_clean":"2023-03-25T01:48:56.988435+0000","last_became_active":"2023-03-25T01:48:47.346812+0000","last_became_peered":"2023-03-25T01:48:47.346812+0000","last_unstale":"2023-03-25T01:48:56.988435+0000","last_undegraded":"2023-03-25T01:48:56.988435+0000","last_fullsized":"2023-03-25T01:48:56.988435+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T01:48:45.924614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T01:48:45.924614+0000","last_clean_scrub_stamp":"2023-03-25T01:48:45.924614+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T07:21:10.392042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"20'30","reported_seq":40,"reported_epoch":20,"state":"active+clean","last_fresh":"2023-03-25T01:49:04.130697+0000","last_change":"2023-03-25T01:48:56.990522+0000","last_active":"2023-03-25T01:49:04.130697+0000","last_peered":"2023-03-25T01:49:04.130697+0000","last_clean":"2023-03-25T01:49:04.130697+0000","last_became_active":"2023-03-25T01:48:56.990276+0000","last_became_peered":"2023-03-25T01:48:56.990276+0000","last_unstale":"2023-03-25T01:49:04.130697+0000","last_undegraded":"2023-03-25T01:49:04.130697+0000","last_fullsized":"2023-03-25T01:49:04.130697+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T01:48:55.982031+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T01:48:55.982031+0000","last_clean_scrub_stamp":"2023-03-25T01:48:55.982031+0000","objects_scrubbed":0,"log_size":30,"log_dups_size":0,"ondisk_log_size":30,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:43:07.120795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":650000,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":30,"num_write_kb":649,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":15,"reported_epoch":19,"state":"active+clean","last_fresh":"2023-03-25T01:48:56.988086+0000","last_change":"2023-03-25T01:48:46.941510+0000","last_active":"2023-03-25T01:48:56.988086+0000","last_peered":"2023-03-25T01:48:56.988086+0000","last_clean":"2023-03-25T01:48:56.988086+0000","last_became_active":"2023-03-25T01:48:46.941266+0000","last_became_peered":"2023-03-25T01:48:46.941266+0000","last_unstale":"2023-03-25T01:48:56.988086+0000","last_undegraded":"2023-03-25T01:48:56.988086+0000","last_fullsized":"2023-03-25T01:48:56.988086+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T01:48:45.924614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T01:48:45.924614+0000","last_clean_scrub_stamp":"2023-03-25T01:48:45.924614+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:34:02.770586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1],"acting":[5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":10,"reported_epoch":19,"state":"active+clean","last_fresh":"2023-03-25T01:48:58.909220+0000","last_change":"2023-03-25T01:48:56.990419+0000","last_active":"2023-03-25T01:48:58.909220+0000","last_peered":"2023-03-25T01:48:58.909220+0000","last_clean":"2023-03-25T01:48:58.909220+0000","last_became_active":"2023-03-25T01:48:56.990188+0000","last_became_peered":"2023-03-25T01:48:56.990188+0000","last_unstale":"2023-03-25T01:48:58.909220+0000","last_undegraded":"2023-03-25T01:48:58.909220+0000","last_fullsized":"2023-03-25T01:48:58.909220+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T01:48:55.982031+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T01:48:55.982031+0000","last_clean_scrub_stamp":"2023-03-25T01:48:55.982031+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:43:07.120795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":14,"reported_epoch":19,"state":"active+clean","last_fresh":"2023-03-25T01:48:57.402752+0000","last_change":"2023-03-25T01:48:46.942191+0000","last_active":"2023-03-25T01:48:57.402752+0000","last_peered":"2023-03-25T01:48:57.402752+0000","last_clean":"2023-03-25T01:48:57.402752+0000","last_became_active":"2023-03-25T01:48:46.941918+0000","last_became_peered":"2023-03-25T01:48:46.941918+0000","last_unstale":"2023-03-25T01:48:57.402752+0000","last_undegraded":"2023-03-25T01:48:57.402752+0000","last_fullsized":"2023-03-25T01:48:57.402752+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T01:48:45.924614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T01:48:45.924614+0000","last_clean_scrub_stamp":"2023-03-25T01:48:45.924614+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T10:36:32.544700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5],"acting":[2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":10,"reported_epoch":20,"state":"active+clean","last_fresh":"2023-03-25T01:48:57.985996+0000","last_change":"2023-03-25T01:48:57.404656+0000","last_active":"2023-03-25T01:48:57.985996+0000","last_peered":"2023-03-25T01:48:57.985996+0000","last_clean":"2023-03-25T01:48:57.985996+0000","last_became_active":"2023-03-25T01:48:57.404416+0000","last_became_peered":"2023-03-25T01:48:57.404416+0000","last_unstale":"2023-03-25T01:48:57.985996+0000","last_undegraded":"2023-03-25T01:48:57.985996+0000","last_fullsized":"2023-03-25T01:48:57.985996+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T01:48:55.982031+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T01:48:55.982031+0000","last_clean_scrub_stamp":"2023-03-25T01:48:55.982031+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T12:21:35.179737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":15,"reported_epoch":19,"state":"active+clean","last_fresh":"2023-03-25T01:48:56.987983+0000","last_change":"2023-03-25T01:48:47.283824+0000","last_active":"2023-03-25T01:48:56.987983+0000","last_peered":"2023-03-25T01:48:56.987983+0000","last_clean":"2023-03-25T01:48:56.987983+0000","last_became_active":"2023-03-25T01:48:47.283534+0000","last_became_peered":"2023-03-25T01:48:47.283534+0000","last_unstale":"2023-03-25T01:48:56.987983+0000","last_undegraded":"2023-03-25T01:48:56.987983+0000","last_fullsized":"2023-03-25T01:48:56.987983+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T01:48:45.924614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T01:48:45.924614+0000","last_clean_scrub_stamp":"2023-03-25T01:48:45.924614+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T10:36:32.544700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4],"acting":[5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"13'97","reported_seq":142,"reported_epoch":19,"state":"active+clean","last_fresh":"2023-03-25T01:48:56.988118+0000","last_change":"2023-03-25T01:48:45.934080+0000","last_active":"2023-03-25T01:48:56.988118+0000","last_peered":"2023-03-25T01:48:56.988118+0000","last_clean":"2023-03-25T01:48:56.988118+0000","last_became_active":"2023-03-25T01:48:45.933746+0000","last_became_peered":"2023-03-25T01:48:45.933746+0000","last_unstale":"2023-03-25T01:48:56.988118+0000","last_undegraded":"2023-03-25T01:48:56.988118+0000","last_fullsized":"2023-03-25T01:48:56.988118+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T01:48:44.920729+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T01:48:44.920729+0000","last_clean_scrub_stamp":"2023-03-25T01:48:44.920729+0000","objects_scrubbed":0,"log_size":97,"log_dups_size":0,"ondisk_log_size":97,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:43:52.243010+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":137,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.2","version":"20'30","reported_seq":40,"reported_epoch":20,"state":"active+clean","last_fresh":"2023-03-25T01:49:06.431894+0000","last_change":"2023-03-25T01:48:56.991149+0000","last_active":"2023-03-25T01:49:06.431894+0000","last_peered":"2023-03-25T01:49:06.431894+0000","last_clean":"2023-03-25T01:49:06.431894+0000","last_became_active":"2023-03-25T01:48:56.990889+0000","last_became_peered":"2023-03-25T01:48:56.990889+0000","last_unstale":"2023-03-25T01:49:06.431894+0000","last_undegraded":"2023-03-25T01:49:06.431894+0000","last_fullsized":"2023-03-25T01:49:06.431894+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T01:48:55.982031+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T01:48:55.982031+0000","last_clean_scrub_stamp":"2023-03-25T01:48:55.982031+0000","objects_scrubbed":0,"log_size":30,"log_dups_size":0,"ondisk_log_size":30,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T10:51:06.108055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":650000,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":30,"num_write_kb":649,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5],"acting":[3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":14,"reported_epoch":19,"state":"active+clean","last_fresh":"2023-03-25T01:48:56.984507+0000","last_change":"2023-03-25T01:48:46.943089+0000","last_active":"2023-03-25T01:48:56.984507+0000","last_peered":"2023-03-25T01:48:56.984507+0000","last_clean":"2023-03-25T01:48:56.984507+0000","last_became_active":"2023-03-25T01:48:46.942843+0000","last_became_peered":"2023-03-25T01:48:46.942843+0000","last_unstale":"2023-03-25T01:48:56.984507+0000","last_undegraded":"2023-03-25T01:48:56.984507+0000","last_fullsized":"2023-03-25T01:48:56.984507+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T01:48:45.924614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T01:48:45.924614+0000","last_clean_scrub_stamp":"2023-03-25T01:48:45.924614+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T07:21:10.392042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5],"acting":[3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1","version":"20'34","reported_seq":44,"reported_epoch":20,"state":"active+clean","last_fresh":"2023-03-25T01:49:13.951565+0000","last_change":"2023-03-25T01:48:56.989334+0000","last_active":"2023-03-25T01:49:13.951565+0000","last_peered":"2023-03-25T01:49:13.951565+0000","last_clean":"2023-03-25T01:49:13.951565+0000","last_became_active":"2023-03-25T01:48:56.989079+0000","last_became_peered":"2023-03-25T01:48:56.989079+0000","last_unstale":"2023-03-25T01:49:13.951565+0000","last_undegraded":"2023-03-25T01:49:13.951565+0000","last_fullsized":"2023-03-25T01:49:13.951565+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T01:48:55.982031+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T01:48:55.982031+0000","last_clean_scrub_stamp":"2023-03-25T01:48:55.982031+0000","objects_scrubbed":0,"log_size":34,"log_dups_size":0,"ondisk_log_size":34,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T07:51:55.253730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":330000,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":34,"num_write_kb":339,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":14,"reported_epoch":19,"state":"active+clean","last_fresh":"2023-03-25T01:48:56.984518+0000","last_change":"2023-03-25T01:48:47.283615+0000","last_active":"2023-03-25T01:48:56.984518+0000","last_peered":"2023-03-25T01:48:56.984518+0000","last_clean":"2023-03-25T01:48:56.984518+0000","last_became_active":"2023-03-25T01:48:47.283359+0000","last_became_peered":"2023-03-25T01:48:47.283359+0000","last_unstale":"2023-03-25T01:48:56.984518+0000","last_undegraded":"2023-03-25T01:48:56.984518+0000","last_fullsized":"2023-03-25T01:48:56.984518+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T01:48:45.924614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T01:48:45.924614+0000","last_clean_scrub_stamp":"2023-03-25T01:48:45.924614+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T02:55:58.247062+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.4","version":"20'5","reported_seq":15,"reported_epoch":20,"state":"active+clean","last_fresh":"2023-03-25T01:48:58.920001+0000","last_change":"2023-03-25T01:48:57.404723+0000","last_active":"2023-03-25T01:48:58.920001+0000","last_peered":"2023-03-25T01:48:58.920001+0000","last_clean":"2023-03-25T01:48:58.920001+0000","last_became_active":"2023-03-25T01:48:57.404409+0000","last_became_peered":"2023-03-25T01:48:57.404409+0000","last_unstale":"2023-03-25T01:48:58.920001+0000","last_undegraded":"2023-03-25T01:48:58.920001+0000","last_fullsized":"2023-03-25T01:48:58.920001+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T01:48:55.982031+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T01:48:55.982031+0000","last_clean_scrub_stamp":"2023-03-25T01:48:55.982031+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T07:51:55.253730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":640000,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5,"num_write_kb":627,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"20'88","reported_seq":98,"reported_epoch":20,"state":"active+clean","last_fresh":"2023-03-25T01:49:18.814701+0000","last_change":"2023-03-25T01:48:56.988823+0000","last_active":"2023-03-25T01:49:18.814701+0000","last_peered":"2023-03-25T01:49:18.814701+0000","last_clean":"2023-03-25T01:49:18.814701+0000","last_became_active":"2023-03-25T01:48:56.988568+0000","last_became_peered":"2023-03-25T01:48:56.988568+0000","last_unstale":"2023-03-25T01:49:18.814701+0000","last_undegraded":"2023-03-25T01:49:18.814701+0000","last_fullsized":"2023-03-25T01:49:18.814701+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T01:48:55.982031+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T01:48:55.982031+0000","last_clean_scrub_stamp":"2023-03-25T01:48:55.982031+0000","objects_scrubbed":0,"log_size":88,"log_dups_size":0,"ondisk_log_size":88,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T10:51:06.108055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1310000,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":88,"num_write_kb":1314,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":14,"reported_epoch":19,"state":"active+clean","last_fresh":"2023-03-25T01:48:56.988499+0000","last_change":"2023-03-25T01:48:47.347152+0000","last_active":"2023-03-25T01:48:56.988499+0000","last_peered":"2023-03-25T01:48:56.988499+0000","last_clean":"2023-03-25T01:48:56.988499+0000","last_became_active":"2023-03-25T01:48:47.346919+0000","last_became_peered":"2023-03-25T01:48:47.346919+0000","last_unstale":"2023-03-25T01:48:56.988499+0000","last_undegraded":"2023-03-25T01:48:56.988499+0000","last_fullsized":"2023-03-25T01:48:56.988499+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T01:48:45.924614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T01:48:45.924614+0000","last_clean_scrub_stamp":"2023-03-25T01:48:45.924614+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T02:55:58.247062+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":14,"reported_epoch":19,"state":"active+clean","last_fresh":"2023-03-25T01:48:56.984528+0000","last_change":"2023-03-25T01:48:47.347110+0000","last_active":"2023-03-25T01:48:56.984528+0000","last_peered":"2023-03-25T01:48:56.984528+0000","last_clean":"2023-03-25T01:48:56.984528+0000","last_became_active":"2023-03-25T01:48:47.346832+0000","last_became_peered":"2023-03-25T01:48:47.346832+0000","last_unstale":"2023-03-25T01:48:56.984528+0000","last_undegraded":"2023-03-25T01:48:56.984528+0000","last_fullsized":"2023-03-25T01:48:56.984528+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T01:48:45.924614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T01:48:45.924614+0000","last_clean_scrub_stamp":"2023-03-25T01:48:45.924614+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:34:02.770586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"20'126","reported_seq":136,"reported_epoch":20,"state":"active+clean","last_fresh":"2023-03-25T01:49:17.626037+0000","last_change":"2023-03-25T01:48:56.990341+0000","last_active":"2023-03-25T01:49:17.626037+0000","last_peered":"2023-03-25T01:49:17.626037+0000","last_clean":"2023-03-25T01:49:17.626037+0000","last_became_active":"2023-03-25T01:48:56.990039+0000","last_became_peered":"2023-03-25T01:48:56.990039+0000","last_unstale":"2023-03-25T01:49:17.626037+0000","last_undegraded":"2023-03-25T01:49:17.626037+0000","last_fullsized":"2023-03-25T01:49:17.626037+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T01:48:55.982031+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T01:48:55.982031+0000","last_clean_scrub_stamp":"2023-03-25T01:48:55.982031+0000","objects_scrubbed":0,"log_size":126,"log_dups_size":0,"ondisk_log_size":126,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T06:14:10.792416+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1630000,"num_objects":5,"num_object_clones":0,"num_object_copies":10,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":126,"num_write_kb":1653,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":5,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":3,"num_pg":12,"stat_sum":{"num_bytes":6510000,"num_objects":20,"num_object_clones":0,"num_object_copies":40,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":20,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":403,"num_write_kb":6544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":19,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":13123584,"data_stored":13020000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":403,"ondisk_log_size":403,"up":24,"acting":24,"num_store_stats":6},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":137,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":97,"ondisk_log_size":97,"up":2,"acting":2,"num_store_stats":2},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":6}],"osd_stats":[{"osd":5,"up_from":11,"seq":47244640265,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":104857600,"kb_used":6608,"kb_used_data":1480,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":104850992,"statfs":{"total":107374182400,"available":107367415808,"internally_reserved":0,"allocated":1515520,"data_stored":1366284,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,3,4],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34999999999999998}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.307}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29399999999999998}]},{"osd":3,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.434}]},{"osd":4,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.36499999999999999}]}]},{"osd":4,"up_from":11,"seq":47244640267,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":104857600,"kb_used":8024,"kb_used_data":2768,"kb_used_omap":0,"kb_used_meta":5248,"kb_avail":104849576,"statfs":{"total":107374182400,"available":107365965824,"internally_reserved":0,"allocated":2834432,"data_stored":2676284,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5373952},"hb_peers":[0,1,2,3,5],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.31900000000000001}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.24099999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.25800000000000001}]},{"osd":3,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.41499999999999998}]},{"osd":5,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.435}]}]},{"osd":3,"up_from":11,"seq":47244640266,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":104857600,"kb_used":8736,"kb_used_data":3416,"kb_used_omap":0,"kb_used_meta":5312,"kb_avail":104848864,"statfs":{"total":107374182400,"available":107365236736,"internally_reserved":0,"allocated":3497984,"data_stored":3336284,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5439488},"hb_peers":[0,1,2,4,5],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.30199999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.35599999999999998}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.38400000000000001}]},{"osd":4,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.312}]},{"osd":5,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.32000000000000001}]}]},{"osd":2,"up_from":11,"seq":47244640266,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":104857600,"kb_used":6916,"kb_used_data":1788,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":104850684,"statfs":{"total":107374182400,"available":107367100416,"internally_reserved":0,"allocated":1830912,"data_stored":1676284,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,3,4,5],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.498}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.45200000000000001}]},{"osd":3,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.52800000000000002}]},{"osd":4,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.32100000000000001}]},{"osd":5,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.58299999999999996}]}]},{"osd":0,"up_from":11,"seq":47244640265,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":104857600,"kb_used":8348,"kb_used_data":2708,"kb_used_omap":0,"kb_used_meta":5632,"kb_avail":104849252,"statfs":{"total":107374182400,"available":107365634048,"internally_reserved":0,"allocated":2772992,"data_stored":2616652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5767168},"hb_peers":[1,2,3,4,5],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.32800000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26000000000000001}]},{"osd":3,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.30399999999999999}]},{"osd":4,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.24399999999999999}]},{"osd":5,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":1,"up_from":11,"seq":47244640266,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":104857600,"kb_used":8656,"kb_used_data":3016,"kb_used_omap":0,"kb_used_meta":5632,"kb_avail":104848944,"statfs":{"total":107374182400,"available":107365318656,"internally_reserved":0,"allocated":3088384,"data_stored":2926652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5767168},"hb_peers":[0,2,3,4,5],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.27300000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34799999999999998}]},{"osd":3,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.24399999999999999}]},{"osd":4,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.36499999999999999}]},{"osd":5,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.49099999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1974272,"data_stored":1960000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2289664,"data_stored":2270000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1626112,"data_stored":1610000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":3293184,"data_stored":3270000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2629632,"data_stored":2610000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1310720,"data_stored":1300000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T02:15:44.816 INFO:tasks.ceph:Checking cluster log for badness...
, 2023-03-25T02:15:44.817 DEBUG:teuthology.orchestra.run.smithi087:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/ceph.log | egrep -v 'overall HEALTH_' | egrep -v '\(OSDMAP_FLAGS\)' | egrep -v '\(OSD_' | egrep -v '\(PG_' | egrep -v '\(TOO_FEW_PGS\)' | egrep -v '\(POOL_APP_NOT_ENABLED\)' | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1
, 2023-03-25T02:15:44.885 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/ceph-0 on ubuntu@smithi087.front.sepia.ceph.com

Ignoring malparsed backtrace: 2023-03-25T02:15:48.639 DEBUG:teuthology.orchestra.run.smithi087:> sudo tar cz -f - -C /home/ubuntu/cephtest/archive -- .
, 2023-03-25T02:15:57.238 INFO:teuthology.task.internal:Transferring binaries for coredumps...
, 2023-03-25T02:15:57.288 INFO:teuthology.task.internal: core looks like: /home/teuthworker/archive/benhanokh-2023-03-16_20:57:19-rados-WIP_GBH_snap_mapper_5-distro-default-smithi/7210931/remote/smithi087/coredump/1679708966.134597.core: ELF 64-bit LSB core file x86-64, version 1 (SYSV), SVR4-style, from 'ceph-objectstore-tool --data-path /var/lib/ceph/osd/ceph-3 --journal-path /var/', real uid: 0, effective uid: 0, real gid: 0, effective gid: 0, execfn: '/bin/ceph-objectstore-tool', platform: 'x86_64'

Ignoring malparsed backtrace:  ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
,  1: /lib64/libpthread.so.0(+0x12cf0) [0x7f2181c7ecf0]
,  2: __pthread_mutex_lock()

Ignoring malparsed backtrace: 2023-03-25T02:01:50.041 INFO:teuthology.orchestra.run.smithi087.stdout:{"pg_ready":true,"pg_map":{"version":408,"stamp":"2023-03-25T02:01:48.868189+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":7100368,"num_objects":22,"num_object_clones":0,"num_object_copies":44,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":540,"num_write_kb":8438,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":19,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":500,"ondisk_log_size":500,"up":42,"acting":42,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":42,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":629145600,"kb_used":47288,"kb_used_data":15176,"kb_used_omap":0,"kb_used_meta":32064,"kb_avail":629098312,"statfs":{"total":644245094400,"available":644196671488,"internally_reserved":0,"allocated":15540224,"data_stored":14598440,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":32833536},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002186"},"pg_stats":[{"pgid":"3.b","version":"20'26","reported_seq":36,"reported_epoch":20,"state":"active+clean","last_fresh":"2023-03-25T01:49:07.722753+0000","last_change":"2023-03-25T01:48:56.990468+0000","last_active":"2023-03-25T01:49:07.722753+0000","last_peered":"2023-03-25T01:49:07.722753+0000","last_clean":"2023-03-25T01:49:07.722753+0000","last_became_active":"2023-03-25T01:48:56.990194+0000","last_became_peered":"2023-03-25T01:48:56.990194+0000","last_unstale":"2023-03-25T01:49:07.722753+0000","last_undegraded":"2023-03-25T01:49:07.722753+0000","last_fullsized":"2023-03-25T01:49:07.722753+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T01:48:55.982031+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T01:48:55.982031+0000","last_clean_scrub_stamp":"2023-03-25T01:48:55.982031+0000","objects_scrubbed":0,"log_size":26,"log_dups_size":0,"ondisk_log_size":26,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T07:51:55.253730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":330000,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":26,"num_write_kb":335,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":10,"reported_epoch":19,"state":"active+clean","last_fresh":"2023-03-25T01:48:58.910560+0000","last_change":"2023-03-25T01:48:56.990354+0000","last_active":"2023-03-25T01:48:58.910560+0000","last_peered":"2023-03-25T01:48:58.910560+0000","last_clean":"2023-03-25T01:48:58.910560+0000","last_became_active":"2023-03-25T01:48:56.989754+0000","last_became_peered":"2023-03-25T01:48:56.989754+0000","last_unstale":"2023-03-25T01:48:58.910560+0000","last_undegraded":"2023-03-25T01:48:58.910560+0000","last_fullsized":"2023-03-25T01:48:58.910560+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T01:48:55.982031+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T01:48:55.982031+0000","last_clean_scrub_stamp":"2023-03-25T01:48:55.982031+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:18:38.285450+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"20'64","reported_seq":74,"reported_epoch":20,"state":"active+clean","last_fresh":"2023-03-25T01:49:12.193662+0000","last_change":"2023-03-25T01:48:57.404560+0000","last_active":"2023-03-25T01:49:12.193662+0000","last_peered":"2023-03-25T01:49:12.193662+0000","last_clean":"2023-03-25T01:49:12.193662+0000","last_became_active":"2023-03-25T01:48:57.404333+0000","last_became_peered":"2023-03-25T01:48:57.404333+0000","last_unstale":"2023-03-25T01:49:12.193662+0000","last_undegraded":"2023-03-25T01:49:12.193662+0000","last_fullsized":"2023-03-25T01:49:12.193662+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T01:48:55.982031+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T01:48:55.982031+0000","last_clean_scrub_stamp":"2023-03-25T01:48:55.982031+0000","objects_scrubbed":0,"log_size":64,"log_dups_size":0,"ondisk_log_size":64,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:18:38.285450+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":970000,"num_objects":3,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":64,"num_write_kb":978,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":3,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2],"acting":[4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":10,"reported_epoch":19,"state":"active+clean","last_fresh":"2023-03-25T01:48:58.909182+0000","last_change":"2023-03-25T01:48:56.989694+0000","last_active":"2023-03-25T01:48:58.909182+0000","last_peered":"2023-03-25T01:48:58.909182+0000","last_clean":"2023-03-25T01:48:58.909182+0000","last_became_active":"2023-03-25T01:48:56.989461+0000","last_became_peered":"2023-03-25T01:48:56.989461+0000","last_unstale":"2023-03-25T01:48:58.909182+0000","last_undegraded":"2023-03-25T01:48:58.909182+0000","last_fullsized":"2023-03-25T01:48:58.909182+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T01:48:55.982031+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T01:48:55.982031+0000","last_clean_scrub_stamp":"2023-03-25T01:48:55.982031+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T12:21:35.179737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":14,"reported_epoch":19,"state":"active+clean","last_fresh":"2023-03-25T01:48:56.988435+0000","last_change":"2023-03-25T01:48:47.347051+0000","last_active":"2023-03-25T01:48:56.988435+0000","last_peered":"2023-03-25T01:48:56.988435+0000","last_clean":"2023-03-25T01:48:56.988435+0000","last_became_active":"2023-03-25T01:48:47.346812+0000","last_became_peered":"2023-03-25T01:48:47.346812+0000","last_unstale":"2023-03-25T01:48:56.988435+0000","last_undegraded":"2023-03-25T01:48:56.988435+0000","last_fullsized":"2023-03-25T01:48:56.988435+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T01:48:45.924614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T01:48:45.924614+0000","last_clean_scrub_stamp":"2023-03-25T01:48:45.924614+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T07:21:10.392042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"20'30","reported_seq":40,"reported_epoch":20,"state":"active+clean","last_fresh":"2023-03-25T01:49:04.130697+0000","last_change":"2023-03-25T01:48:56.990522+0000","last_active":"2023-03-25T01:49:04.130697+0000","last_peered":"2023-03-25T01:49:04.130697+0000","last_clean":"2023-03-25T01:49:04.130697+0000","last_became_active":"2023-03-25T01:48:56.990276+0000","last_became_peered":"2023-03-25T01:48:56.990276+0000","last_unstale":"2023-03-25T01:49:04.130697+0000","last_undegraded":"2023-03-25T01:49:04.130697+0000","last_fullsized":"2023-03-25T01:49:04.130697+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T01:48:55.982031+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T01:48:55.982031+0000","last_clean_scrub_stamp":"2023-03-25T01:48:55.982031+0000","objects_scrubbed":0,"log_size":30,"log_dups_size":0,"ondisk_log_size":30,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:43:07.120795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":650000,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":30,"num_write_kb":649,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":15,"reported_epoch":19,"state":"active+clean","last_fresh":"2023-03-25T01:48:56.988086+0000","last_change":"2023-03-25T01:48:46.941510+0000","last_active":"2023-03-25T01:48:56.988086+0000","last_peered":"2023-03-25T01:48:56.988086+0000","last_clean":"2023-03-25T01:48:56.988086+0000","last_became_active":"2023-03-25T01:48:46.941266+0000","last_became_peered":"2023-03-25T01:48:46.941266+0000","last_unstale":"2023-03-25T01:48:56.988086+0000","last_undegraded":"2023-03-25T01:48:56.988086+0000","last_fullsized":"2023-03-25T01:48:56.988086+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T01:48:45.924614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T01:48:45.924614+0000","last_clean_scrub_stamp":"2023-03-25T01:48:45.924614+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:34:02.770586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1],"acting":[5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":10,"reported_epoch":19,"state":"active+clean","last_fresh":"2023-03-25T01:48:58.909220+0000","last_change":"2023-03-25T01:48:56.990419+0000","last_active":"2023-03-25T01:48:58.909220+0000","last_peered":"2023-03-25T01:48:58.909220+0000","last_clean":"2023-03-25T01:48:58.909220+0000","last_became_active":"2023-03-25T01:48:56.990188+0000","last_became_peered":"2023-03-25T01:48:56.990188+0000","last_unstale":"2023-03-25T01:48:58.909220+0000","last_undegraded":"2023-03-25T01:48:58.909220+0000","last_fullsized":"2023-03-25T01:48:58.909220+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T01:48:55.982031+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T01:48:55.982031+0000","last_clean_scrub_stamp":"2023-03-25T01:48:55.982031+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:43:07.120795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":14,"reported_epoch":19,"state":"active+clean","last_fresh":"2023-03-25T01:48:57.402752+0000","last_change":"2023-03-25T01:48:46.942191+0000","last_active":"2023-03-25T01:48:57.402752+0000","last_peered":"2023-03-25T01:48:57.402752+0000","last_clean":"2023-03-25T01:48:57.402752+0000","last_became_active":"2023-03-25T01:48:46.941918+0000","last_became_peered":"2023-03-25T01:48:46.941918+0000","last_unstale":"2023-03-25T01:48:57.402752+0000","last_undegraded":"2023-03-25T01:48:57.402752+0000","last_fullsized":"2023-03-25T01:48:57.402752+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T01:48:45.924614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T01:48:45.924614+0000","last_clean_scrub_stamp":"2023-03-25T01:48:45.924614+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T10:36:32.544700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5],"acting":[2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":10,"reported_epoch":20,"state":"active+clean","last_fresh":"2023-03-25T01:48:57.985996+0000","last_change":"2023-03-25T01:48:57.404656+0000","last_active":"2023-03-25T01:48:57.985996+0000","last_peered":"2023-03-25T01:48:57.985996+0000","last_clean":"2023-03-25T01:48:57.985996+0000","last_became_active":"2023-03-25T01:48:57.404416+0000","last_became_peered":"2023-03-25T01:48:57.404416+0000","last_unstale":"2023-03-25T01:48:57.985996+0000","last_undegraded":"2023-03-25T01:48:57.985996+0000","last_fullsized":"2023-03-25T01:48:57.985996+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T01:48:55.982031+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T01:48:55.982031+0000","last_clean_scrub_stamp":"2023-03-25T01:48:55.982031+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T12:21:35.179737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":15,"reported_epoch":19,"state":"active+clean","last_fresh":"2023-03-25T01:48:56.987983+0000","last_change":"2023-03-25T01:48:47.283824+0000","last_active":"2023-03-25T01:48:56.987983+0000","last_peered":"2023-03-25T01:48:56.987983+0000","last_clean":"2023-03-25T01:48:56.987983+0000","last_became_active":"2023-03-25T01:48:47.283534+0000","last_became_peered":"2023-03-25T01:48:47.283534+0000","last_unstale":"2023-03-25T01:48:56.987983+0000","last_undegraded":"2023-03-25T01:48:56.987983+0000","last_fullsized":"2023-03-25T01:48:56.987983+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T01:48:45.924614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T01:48:45.924614+0000","last_clean_scrub_stamp":"2023-03-25T01:48:45.924614+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T10:36:32.544700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4],"acting":[5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"13'97","reported_seq":142,"reported_epoch":19,"state":"active+clean","last_fresh":"2023-03-25T01:48:56.988118+0000","last_change":"2023-03-25T01:48:45.934080+0000","last_active":"2023-03-25T01:48:56.988118+0000","last_peered":"2023-03-25T01:48:56.988118+0000","last_clean":"2023-03-25T01:48:56.988118+0000","last_became_active":"2023-03-25T01:48:45.933746+0000","last_became_peered":"2023-03-25T01:48:45.933746+0000","last_unstale":"2023-03-25T01:48:56.988118+0000","last_undegraded":"2023-03-25T01:48:56.988118+0000","last_fullsized":"2023-03-25T01:48:56.988118+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T01:48:44.920729+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T01:48:44.920729+0000","last_clean_scrub_stamp":"2023-03-25T01:48:44.920729+0000","objects_scrubbed":0,"log_size":97,"log_dups_size":0,"ondisk_log_size":97,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:43:52.243010+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":137,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.2","version":"20'30","reported_seq":40,"reported_epoch":20,"state":"active+clean","last_fresh":"2023-03-25T01:49:06.431894+0000","last_change":"2023-03-25T01:48:56.991149+0000","last_active":"2023-03-25T01:49:06.431894+0000","last_peered":"2023-03-25T01:49:06.431894+0000","last_clean":"2023-03-25T01:49:06.431894+0000","last_became_active":"2023-03-25T01:48:56.990889+0000","last_became_peered":"2023-03-25T01:48:56.990889+0000","last_unstale":"2023-03-25T01:49:06.431894+0000","last_undegraded":"2023-03-25T01:49:06.431894+0000","last_fullsized":"2023-03-25T01:49:06.431894+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T01:48:55.982031+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T01:48:55.982031+0000","last_clean_scrub_stamp":"2023-03-25T01:48:55.982031+0000","objects_scrubbed":0,"log_size":30,"log_dups_size":0,"ondisk_log_size":30,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T10:51:06.108055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":650000,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":30,"num_write_kb":649,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5],"acting":[3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":14,"reported_epoch":19,"state":"active+clean","last_fresh":"2023-03-25T01:48:56.984507+0000","last_change":"2023-03-25T01:48:46.943089+0000","last_active":"2023-03-25T01:48:56.984507+0000","last_peered":"2023-03-25T01:48:56.984507+0000","last_clean":"2023-03-25T01:48:56.984507+0000","last_became_active":"2023-03-25T01:48:46.942843+0000","last_became_peered":"2023-03-25T01:48:46.942843+0000","last_unstale":"2023-03-25T01:48:56.984507+0000","last_undegraded":"2023-03-25T01:48:56.984507+0000","last_fullsized":"2023-03-25T01:48:56.984507+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T01:48:45.924614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T01:48:45.924614+0000","last_clean_scrub_stamp":"2023-03-25T01:48:45.924614+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T07:21:10.392042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5],"acting":[3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1","version":"20'34","reported_seq":44,"reported_epoch":20,"state":"active+clean","last_fresh":"2023-03-25T01:49:13.951565+0000","last_change":"2023-03-25T01:48:56.989334+0000","last_active":"2023-03-25T01:49:13.951565+0000","last_peered":"2023-03-25T01:49:13.951565+0000","last_clean":"2023-03-25T01:49:13.951565+0000","last_became_active":"2023-03-25T01:48:56.989079+0000","last_became_peered":"2023-03-25T01:48:56.989079+0000","last_unstale":"2023-03-25T01:49:13.951565+0000","last_undegraded":"2023-03-25T01:49:13.951565+0000","last_fullsized":"2023-03-25T01:49:13.951565+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T01:48:55.982031+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T01:48:55.982031+0000","last_clean_scrub_stamp":"2023-03-25T01:48:55.982031+0000","objects_scrubbed":0,"log_size":34,"log_dups_size":0,"ondisk_log_size":34,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T07:51:55.253730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":330000,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":34,"num_write_kb":339,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":14,"reported_epoch":19,"state":"active+clean","last_fresh":"2023-03-25T01:48:56.984518+0000","last_change":"2023-03-25T01:48:47.283615+0000","last_active":"2023-03-25T01:48:56.984518+0000","last_peered":"2023-03-25T01:48:56.984518+0000","last_clean":"2023-03-25T01:48:56.984518+0000","last_became_active":"2023-03-25T01:48:47.283359+0000","last_became_peered":"2023-03-25T01:48:47.283359+0000","last_unstale":"2023-03-25T01:48:56.984518+0000","last_undegraded":"2023-03-25T01:48:56.984518+0000","last_fullsized":"2023-03-25T01:48:56.984518+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T01:48:45.924614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T01:48:45.924614+0000","last_clean_scrub_stamp":"2023-03-25T01:48:45.924614+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T02:55:58.247062+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.4","version":"20'5","reported_seq":15,"reported_epoch":20,"state":"active+clean","last_fresh":"2023-03-25T01:48:58.920001+0000","last_change":"2023-03-25T01:48:57.404723+0000","last_active":"2023-03-25T01:48:58.920001+0000","last_peered":"2023-03-25T01:48:58.920001+0000","last_clean":"2023-03-25T01:48:58.920001+0000","last_became_active":"2023-03-25T01:48:57.404409+0000","last_became_peered":"2023-03-25T01:48:57.404409+0000","last_unstale":"2023-03-25T01:48:58.920001+0000","last_undegraded":"2023-03-25T01:48:58.920001+0000","last_fullsized":"2023-03-25T01:48:58.920001+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T01:48:55.982031+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T01:48:55.982031+0000","last_clean_scrub_stamp":"2023-03-25T01:48:55.982031+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T07:51:55.253730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":640000,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5,"num_write_kb":627,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"20'88","reported_seq":98,"reported_epoch":20,"state":"active+clean","last_fresh":"2023-03-25T01:49:18.814701+0000","last_change":"2023-03-25T01:48:56.988823+0000","last_active":"2023-03-25T01:49:18.814701+0000","last_peered":"2023-03-25T01:49:18.814701+0000","last_clean":"2023-03-25T01:49:18.814701+0000","last_became_active":"2023-03-25T01:48:56.988568+0000","last_became_peered":"2023-03-25T01:48:56.988568+0000","last_unstale":"2023-03-25T01:49:18.814701+0000","last_undegraded":"2023-03-25T01:49:18.814701+0000","last_fullsized":"2023-03-25T01:49:18.814701+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T01:48:55.982031+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T01:48:55.982031+0000","last_clean_scrub_stamp":"2023-03-25T01:48:55.982031+0000","objects_scrubbed":0,"log_size":88,"log_dups_size":0,"ondisk_log_size":88,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T10:51:06.108055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1310000,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":88,"num_write_kb":1314,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":14,"reported_epoch":19,"state":"active+clean","last_fresh":"2023-03-25T01:48:56.988499+0000","last_change":"2023-03-25T01:48:47.347152+0000","last_active":"2023-03-25T01:48:56.988499+0000","last_peered":"2023-03-25T01:48:56.988499+0000","last_clean":"2023-03-25T01:48:56.988499+0000","last_became_active":"2023-03-25T01:48:47.346919+0000","last_became_peered":"2023-03-25T01:48:47.346919+0000","last_unstale":"2023-03-25T01:48:56.988499+0000","last_undegraded":"2023-03-25T01:48:56.988499+0000","last_fullsized":"2023-03-25T01:48:56.988499+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T01:48:45.924614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T01:48:45.924614+0000","last_clean_scrub_stamp":"2023-03-25T01:48:45.924614+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T02:55:58.247062+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":14,"reported_epoch":19,"state":"active+clean","last_fresh":"2023-03-25T01:48:56.984528+0000","last_change":"2023-03-25T01:48:47.347110+0000","last_active":"2023-03-25T01:48:56.984528+0000","last_peered":"2023-03-25T01:48:56.984528+0000","last_clean":"2023-03-25T01:48:56.984528+0000","last_became_active":"2023-03-25T01:48:47.346832+0000","last_became_peered":"2023-03-25T01:48:47.346832+0000","last_unstale":"2023-03-25T01:48:56.984528+0000","last_undegraded":"2023-03-25T01:48:56.984528+0000","last_fullsized":"2023-03-25T01:48:56.984528+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T01:48:45.924614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T01:48:45.924614+0000","last_clean_scrub_stamp":"2023-03-25T01:48:45.924614+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:34:02.770586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"20'126","reported_seq":136,"reported_epoch":20,"state":"active+clean","last_fresh":"2023-03-25T01:49:17.626037+0000","last_change":"2023-03-25T01:48:56.990341+0000","last_active":"2023-03-25T01:49:17.626037+0000","last_peered":"2023-03-25T01:49:17.626037+0000","last_clean":"2023-03-25T01:49:17.626037+0000","last_became_active":"2023-03-25T01:48:56.990039+0000","last_became_peered":"2023-03-25T01:48:56.990039+0000","last_unstale":"2023-03-25T01:49:17.626037+0000","last_undegraded":"2023-03-25T01:49:17.626037+0000","last_fullsized":"2023-03-25T01:49:17.626037+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T01:48:55.982031+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T01:48:55.982031+0000","last_clean_scrub_stamp":"2023-03-25T01:48:55.982031+0000","objects_scrubbed":0,"log_size":126,"log_dups_size":0,"ondisk_log_size":126,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T06:14:10.792416+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1630000,"num_objects":5,"num_object_clones":0,"num_object_copies":10,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":126,"num_write_kb":1653,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":5,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":3,"num_pg":12,"stat_sum":{"num_bytes":6510000,"num_objects":20,"num_object_clones":0,"num_object_copies":40,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":20,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":403,"num_write_kb":6544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":19,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":13123584,"data_stored":13020000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":403,"ondisk_log_size":403,"up":24,"acting":24,"num_store_stats":6},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":137,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":97,"ondisk_log_size":97,"up":2,"acting":2,"num_store_stats":2},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":6}],"osd_stats":[{"osd":5,"up_from":11,"seq":47244640265,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":104857600,"kb_used":6608,"kb_used_data":1480,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":104850992,"statfs":{"total":107374182400,"available":107367415808,"internally_reserved":0,"allocated":1515520,"data_stored":1366284,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,3,4],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34999999999999998}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.307}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29399999999999998}]},{"osd":3,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.434}]},{"osd":4,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.36499999999999999}]}]},{"osd":4,"up_from":11,"seq":47244640267,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":104857600,"kb_used":8024,"kb_used_data":2768,"kb_used_omap":0,"kb_used_meta":5248,"kb_avail":104849576,"statfs":{"total":107374182400,"available":107365965824,"internally_reserved":0,"allocated":2834432,"data_stored":2676284,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5373952},"hb_peers":[0,1,2,3,5],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.31900000000000001}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.24099999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.25800000000000001}]},{"osd":3,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.41499999999999998}]},{"osd":5,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.435}]}]},{"osd":3,"up_from":11,"seq":47244640266,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":104857600,"kb_used":8736,"kb_used_data":3416,"kb_used_omap":0,"kb_used_meta":5312,"kb_avail":104848864,"statfs":{"total":107374182400,"available":107365236736,"internally_reserved":0,"allocated":3497984,"data_stored":3336284,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5439488},"hb_peers":[0,1,2,4,5],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.30199999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.35599999999999998}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.38400000000000001}]},{"osd":4,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.312}]},{"osd":5,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.32000000000000001}]}]},{"osd":2,"up_from":11,"seq":47244640266,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":104857600,"kb_used":6916,"kb_used_data":1788,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":104850684,"statfs":{"total":107374182400,"available":107367100416,"internally_reserved":0,"allocated":1830912,"data_stored":1676284,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,3,4,5],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.498}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.45200000000000001}]},{"osd":3,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.52800000000000002}]},{"osd":4,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.32100000000000001}]},{"osd":5,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.58299999999999996}]}]},{"osd":0,"up_from":11,"seq":47244640265,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":104857600,"kb_used":8348,"kb_used_data":2708,"kb_used_omap":0,"kb_used_meta":5632,"kb_avail":104849252,"statfs":{"total":107374182400,"available":107365634048,"internally_reserved":0,"allocated":2772992,"data_stored":2616652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5767168},"hb_peers":[1,2,3,4,5],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.32800000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26000000000000001}]},{"osd":3,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.30399999999999999}]},{"osd":4,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.24399999999999999}]},{"osd":5,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":1,"up_from":11,"seq":47244640266,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":104857600,"kb_used":8656,"kb_used_data":3016,"kb_used_omap":0,"kb_used_meta":5632,"kb_avail":104848944,"statfs":{"total":107374182400,"available":107365318656,"internally_reserved":0,"allocated":3088384,"data_stored":2926652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5767168},"hb_peers":[0,2,3,4,5],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.27300000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34799999999999998}]},{"osd":3,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.24399999999999999}]},{"osd":4,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.36499999999999999}]},{"osd":5,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.49099999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1974272,"data_stored":1960000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2289664,"data_stored":2270000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1626112,"data_stored":1610000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":3293184,"data_stored":3270000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2629632,"data_stored":2610000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1310720,"data_stored":1300000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T02:01:50.044 INFO:tasks.ceph:pgid 3.b last_scrub_stamp 2023-03-25T01:48:55.982031+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=48, tm_sec=55, tm_wday=5, tm_yday=84, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=49, tm_sec=28, tm_wday=5, tm_yday=84, tm_isdst=0)
, 2023-03-25T02:01:50.044 INFO:tasks.ceph:pgid 3.a last_scrub_stamp 2023-03-25T01:48:55.982031+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=48, tm_sec=55, tm_wday=5, tm_yday=84, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=49, tm_sec=28, tm_wday=5, tm_yday=84, tm_isdst=0)

Ignoring malparsed backtrace: 2023-03-25T02:03:11.095 DEBUG:teuthology.orchestra.run.smithi087:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T02:03:11.419 INFO:teuthology.orchestra.run.smithi087.stdout:
, dumped all

Ignoring malparsed backtrace: 2023-03-25T02:03:58.964 INFO:tasks.ceph:pgid 3.8 last_scrub_stamp 2023-03-25T01:48:55.982031+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=48, tm_sec=55, tm_wday=5, tm_yday=84, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=49, tm_sec=28, tm_wday=5, tm_yday=84, tm_isdst=0)
, 2023-03-25T02:03:58.964 INFO:tasks.ceph:pgid 2.4 last_scrub_stamp 2023-03-25T01:48:45.924614+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=48, tm_sec=45, tm_wday=5, tm_yday=84, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=49, tm_sec=28, tm_wday=5, tm_yday=84, tm_isdst=0)
, 2023-03-25T02:03:58.964 INFO:tasks.ceph:pgid 3.5 last_scrub_stamp 2023-03-25T01:48:55.982031+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=48, tm_sec=55, tm_wday=5, tm_yday=84, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=49, tm_sec=28, tm_wday=5, tm_yday=84, tm_isdst=0)

Ignoring malparsed backtrace: 2023-03-25T02:05:20.356 INFO:tasks.ceph:pgid 3.b last_scrub_stamp 2023-03-25T01:48:55.982031+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=48, tm_sec=55, tm_wday=5, tm_yday=84, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=49, tm_sec=28, tm_wday=5, tm_yday=84, tm_isdst=0)
, 2023-03-25T02:05:20.356 INFO:tasks.ceph:pgid 3.a last_scrub_stamp 2023-03-25T01:48:55.982031+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=48, tm_sec=55, tm_wday=5, tm_yday=84, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=49, tm_sec=28, tm_wday=5, tm_yday=84, tm_isdst=0)
, 2023-03-25T02:05:20.356 INFO:tasks.ceph:pgid 3.9 last_scrub_stamp 2023-03-25T01:48:55.982031+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=48, tm_sec=55, tm_wday=5, tm_yday=84, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=49, tm_sec=28, tm_wday=5, tm_yday=84, tm_isdst=0)

Ignoring malparsed backtrace: 2023-03-25T02:06:07.907 INFO:tasks.ceph:pgid 3.3 last_scrub_stamp 2023-03-25T01:48:55.982031+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=48, tm_sec=55, tm_wday=5, tm_yday=84, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=49, tm_sec=28, tm_wday=5, tm_yday=84, tm_isdst=0)
, 2023-03-25T02:06:07.907 INFO:tasks.ceph:pgid 2.1 last_scrub_stamp 2023-03-25T01:48:45.924614+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=48, tm_sec=45, tm_wday=5, tm_yday=84, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=49, tm_sec=28, tm_wday=5, tm_yday=84, tm_isdst=0)
, 2023-03-25T02:06:07.907 INFO:tasks.ceph:pgid 3.0 last_scrub_stamp 2023-03-25T01:48:55.982031+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=48, tm_sec=55, tm_wday=5, tm_yday=84, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=49, tm_sec=28, tm_wday=5, tm_yday=84, tm_isdst=0)

Ignoring malparsed backtrace: 2023-03-25T02:07:29.309 INFO:tasks.ceph:pgid 2.4 last_scrub_stamp 2023-03-25T01:48:45.924614+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=48, tm_sec=45, tm_wday=5, tm_yday=84, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=49, tm_sec=28, tm_wday=5, tm_yday=84, tm_isdst=0)
, 2023-03-25T02:07:29.309 INFO:tasks.ceph:pgid 3.5 last_scrub_stamp 2023-03-25T01:48:55.982031+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=48, tm_sec=55, tm_wday=5, tm_yday=84, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=49, tm_sec=28, tm_wday=5, tm_yday=84, tm_isdst=0)
, 2023-03-25T02:07:29.309 INFO:tasks.ceph:pgid 2.2 last_scrub_stamp 2023-03-25T01:48:45.924614+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=48, tm_sec=45, tm_wday=5, tm_yday=84, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=49, tm_sec=28, tm_wday=5, tm_yday=84, tm_isdst=0)

Ignoring malparsed backtrace: 2023-03-25T02:08:16.851 INFO:tasks.ceph:pgid 1.0 last_scrub_stamp 2023-03-25T01:48:44.920729+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=48, tm_sec=44, tm_wday=5, tm_yday=84, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=49, tm_sec=28, tm_wday=5, tm_yday=84, tm_isdst=0)
, 2023-03-25T02:08:16.851 INFO:tasks.ceph:pgid 3.2 last_scrub_stamp 2023-03-25T01:48:55.982031+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=48, tm_sec=55, tm_wday=5, tm_yday=84, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=49, tm_sec=28, tm_wday=5, tm_yday=84, tm_isdst=0)
, 2023-03-25T02:08:16.852 INFO:tasks.ceph:pgid 2.0 last_scrub_stamp 2023-03-25T01:48:45.924614+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=48, tm_sec=45, tm_wday=5, tm_yday=84, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=49, tm_sec=28, tm_wday=5, tm_yday=84, tm_isdst=0)

Ignoring malparsed backtrace: 2023-03-25T02:09:38.242 INFO:tasks.ceph:pgid 2.1 last_scrub_stamp 2023-03-25T01:48:45.924614+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=48, tm_sec=45, tm_wday=5, tm_yday=84, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=49, tm_sec=28, tm_wday=5, tm_yday=84, tm_isdst=0)
, 2023-03-25T02:09:38.242 INFO:tasks.ceph:pgid 3.0 last_scrub_stamp 2023-03-25T01:48:55.982031+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=48, tm_sec=55, tm_wday=5, tm_yday=84, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=49, tm_sec=28, tm_wday=5, tm_yday=84, tm_isdst=0)
, 2023-03-25T02:09:38.242 INFO:tasks.ceph:pgid 2.3 last_scrub_stamp 2023-03-25T01:48:45.924614+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=48, tm_sec=45, tm_wday=5, tm_yday=84, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=49, tm_sec=28, tm_wday=5, tm_yday=84, tm_isdst=0)

Ignoring malparsed backtrace: 2023-03-25T02:10:25.793 INFO:tasks.ceph:pgid 2.5 last_scrub_stamp 2023-03-25T01:48:45.924614+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=48, tm_sec=45, tm_wday=5, tm_yday=84, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=49, tm_sec=28, tm_wday=5, tm_yday=84, tm_isdst=0)
, 2023-03-25T02:10:25.793 INFO:tasks.ceph:pgid 3.4 last_scrub_stamp 2023-03-25T01:48:55.982031+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=48, tm_sec=55, tm_wday=5, tm_yday=84, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=49, tm_sec=28, tm_wday=5, tm_yday=84, tm_isdst=0)
, 2023-03-25T02:10:25.793 INFO:tasks.ceph:pgid 3.7 last_scrub_stamp 2023-03-25T01:48:55.982031+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=48, tm_sec=55, tm_wday=5, tm_yday=84, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=49, tm_sec=28, tm_wday=5, tm_yday=84, tm_isdst=0)

Ignoring malparsed backtrace: 2023-03-25T02:11:47.190 INFO:tasks.ceph:pgid 3.2 last_scrub_stamp 2023-03-25T01:48:55.982031+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=48, tm_sec=55, tm_wday=5, tm_yday=84, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=49, tm_sec=28, tm_wday=5, tm_yday=84, tm_isdst=0)
, 2023-03-25T02:11:47.190 INFO:tasks.ceph:pgid 2.0 last_scrub_stamp 2023-03-25T01:48:45.924614+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=48, tm_sec=45, tm_wday=5, tm_yday=84, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=49, tm_sec=28, tm_wday=5, tm_yday=84, tm_isdst=0)
, 2023-03-25T02:11:47.190 INFO:tasks.ceph:pgid 3.1 last_scrub_stamp 2023-03-25T01:48:55.982031+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=48, tm_sec=55, tm_wday=5, tm_yday=84, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=49, tm_sec=28, tm_wday=5, tm_yday=84, tm_isdst=0)

Ignoring malparsed backtrace: 2023-03-25T02:12:34.721 INFO:tasks.ceph:pgid 2.7 last_scrub_stamp 2023-03-25T01:48:45.924614+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=48, tm_sec=45, tm_wday=5, tm_yday=84, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=49, tm_sec=28, tm_wday=5, tm_yday=84, tm_isdst=0)
, 2023-03-25T02:12:34.722 INFO:tasks.ceph:pgid 3.6 last_scrub_stamp 2023-03-25T01:48:55.982031+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=48, tm_sec=55, tm_wday=5, tm_yday=84, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=49, tm_sec=28, tm_wday=5, tm_yday=84, tm_isdst=0)
, 2023-03-25T02:12:34.722 INFO:tasks.ceph:Still waiting for all pgs to be scrubbed.

Ignoring malparsed backtrace: 2023-03-25T02:13:56.118 INFO:tasks.ceph:pgid 3.4 last_scrub_stamp 2023-03-25T01:48:55.982031+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=48, tm_sec=55, tm_wday=5, tm_yday=84, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=49, tm_sec=28, tm_wday=5, tm_yday=84, tm_isdst=0)
, 2023-03-25T02:13:56.119 INFO:tasks.ceph:pgid 3.7 last_scrub_stamp 2023-03-25T01:48:55.982031+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=48, tm_sec=55, tm_wday=5, tm_yday=84, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=49, tm_sec=28, tm_wday=5, tm_yday=84, tm_isdst=0)
, 2023-03-25T02:13:56.119 INFO:tasks.ceph:pgid 2.6 last_scrub_stamp 2023-03-25T01:48:45.924614+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=48, tm_sec=45, tm_wday=5, tm_yday=84, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=25, tm_hour=1, tm_min=49, tm_sec=28, tm_wday=5, tm_yday=84, tm_isdst=0)

Ignoring malparsed backtrace: 2023-03-25T02:15:03.973 INFO:teuthology.orchestra.run.smithi087.stdout:
, dumped all
, 2023-03-25T02:15:03.986 INFO:teuthology.orchestra.run.smithi087.stdout:{"pg_ready":true,"pg_map":{"version":805,"stamp":"2023-03-25T02:15:03.017634+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":7100368,"num_objects":22,"num_object_clones":0,"num_object_copies":44,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":540,"num_write_kb":8438,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":19,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":500,"ondisk_log_size":500,"up":42,"acting":42,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":42,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":629145600,"kb_used":47288,"kb_used_data":15176,"kb_used_omap":0,"kb_used_meta":32064,"kb_avail":629098312,"statfs":{"total":644245094400,"available":644196671488,"internally_reserved":0,"allocated":15540224,"data_stored":14598440,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":32833536},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002272"},"pg_stats":[{"pgid":"3.b","version":"20'26","reported_seq":36,"reported_epoch":20,"state":"active+clean","last_fresh":"2023-03-25T01:49:07.722753+0000","last_change":"2023-03-25T01:48:56.990468+0000","last_active":"2023-03-25T01:49:07.722753+0000","last_peered":"2023-03-25T01:49:07.722753+0000","last_clean":"2023-03-25T01:49:07.722753+0000","last_became_active":"2023-03-25T01:48:56.990194+0000","last_became_peered":"2023-03-25T01:48:56.990194+0000","last_unstale":"2023-03-25T01:49:07.722753+0000","last_undegraded":"2023-03-25T01:49:07.722753+0000","last_fullsized":"2023-03-25T01:49:07.722753+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T01:48:55.982031+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T01:48:55.982031+0000","last_clean_scrub_stamp":"2023-03-25T01:48:55.982031+0000","objects_scrubbed":0,"log_size":26,"log_dups_size":0,"ondisk_log_size":26,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T07:51:55.253730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":330000,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":26,"num_write_kb":335,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":10,"reported_epoch":19,"state":"active+clean","last_fresh":"2023-03-25T01:48:58.910560+0000","last_change":"2023-03-25T01:48:56.990354+0000","last_active":"2023-03-25T01:48:58.910560+0000","last_peered":"2023-03-25T01:48:58.910560+0000","last_clean":"2023-03-25T01:48:58.910560+0000","last_became_active":"2023-03-25T01:48:56.989754+0000","last_became_peered":"2023-03-25T01:48:56.989754+0000","last_unstale":"2023-03-25T01:48:58.910560+0000","last_undegraded":"2023-03-25T01:48:58.910560+0000","last_fullsized":"2023-03-25T01:48:58.910560+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T01:48:55.982031+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T01:48:55.982031+0000","last_clean_scrub_stamp":"2023-03-25T01:48:55.982031+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:18:38.285450+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"20'64","reported_seq":74,"reported_epoch":20,"state":"active+clean","last_fresh":"2023-03-25T01:49:12.193662+0000","last_change":"2023-03-25T01:48:57.404560+0000","last_active":"2023-03-25T01:49:12.193662+0000","last_peered":"2023-03-25T01:49:12.193662+0000","last_clean":"2023-03-25T01:49:12.193662+0000","last_became_active":"2023-03-25T01:48:57.404333+0000","last_became_peered":"2023-03-25T01:48:57.404333+0000","last_unstale":"2023-03-25T01:49:12.193662+0000","last_undegraded":"2023-03-25T01:49:12.193662+0000","last_fullsized":"2023-03-25T01:49:12.193662+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T01:48:55.982031+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T01:48:55.982031+0000","last_clean_scrub_stamp":"2023-03-25T01:48:55.982031+0000","objects_scrubbed":0,"log_size":64,"log_dups_size":0,"ondisk_log_size":64,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:18:38.285450+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":970000,"num_objects":3,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":64,"num_write_kb":978,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":3,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2],"acting":[4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":10,"reported_epoch":19,"state":"active+clean","last_fresh":"2023-03-25T01:48:58.909182+0000","last_change":"2023-03-25T01:48:56.989694+0000","last_active":"2023-03-25T01:48:58.909182+0000","last_peered":"2023-03-25T01:48:58.909182+0000","last_clean":"2023-03-25T01:48:58.909182+0000","last_became_active":"2023-03-25T01:48:56.989461+0000","last_became_peered":"2023-03-25T01:48:56.989461+0000","last_unstale":"2023-03-25T01:48:58.909182+0000","last_undegraded":"2023-03-25T01:48:58.909182+0000","last_fullsized":"2023-03-25T01:48:58.909182+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T01:48:55.982031+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T01:48:55.982031+0000","last_clean_scrub_stamp":"2023-03-25T01:48:55.982031+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T12:21:35.179737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":14,"reported_epoch":19,"state":"active+clean","last_fresh":"2023-03-25T01:48:56.988435+0000","last_change":"2023-03-25T01:48:47.347051+0000","last_active":"2023-03-25T01:48:56.988435+0000","last_peered":"2023-03-25T01:48:56.988435+0000","last_clean":"2023-03-25T01:48:56.988435+0000","last_became_active":"2023-03-25T01:48:47.346812+0000","last_became_peered":"2023-03-25T01:48:47.346812+0000","last_unstale":"2023-03-25T01:48:56.988435+0000","last_undegraded":"2023-03-25T01:48:56.988435+0000","last_fullsized":"2023-03-25T01:48:56.988435+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T01:48:45.924614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T01:48:45.924614+0000","last_clean_scrub_stamp":"2023-03-25T01:48:45.924614+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T07:21:10.392042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"20'30","reported_seq":40,"reported_epoch":20,"state":"active+clean","last_fresh":"2023-03-25T01:49:04.130697+0000","last_change":"2023-03-25T01:48:56.990522+0000","last_active":"2023-03-25T01:49:04.130697+0000","last_peered":"2023-03-25T01:49:04.130697+0000","last_clean":"2023-03-25T01:49:04.130697+0000","last_became_active":"2023-03-25T01:48:56.990276+0000","last_became_peered":"2023-03-25T01:48:56.990276+0000","last_unstale":"2023-03-25T01:49:04.130697+0000","last_undegraded":"2023-03-25T01:49:04.130697+0000","last_fullsized":"2023-03-25T01:49:04.130697+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T01:48:55.982031+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T01:48:55.982031+0000","last_clean_scrub_stamp":"2023-03-25T01:48:55.982031+0000","objects_scrubbed":0,"log_size":30,"log_dups_size":0,"ondisk_log_size":30,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:43:07.120795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":650000,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":30,"num_write_kb":649,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":15,"reported_epoch":19,"state":"active+clean","last_fresh":"2023-03-25T01:48:56.988086+0000","last_change":"2023-03-25T01:48:46.941510+0000","last_active":"2023-03-25T01:48:56.988086+0000","last_peered":"2023-03-25T01:48:56.988086+0000","last_clean":"2023-03-25T01:48:56.988086+0000","last_became_active":"2023-03-25T01:48:46.941266+0000","last_became_peered":"2023-03-25T01:48:46.941266+0000","last_unstale":"2023-03-25T01:48:56.988086+0000","last_undegraded":"2023-03-25T01:48:56.988086+0000","last_fullsized":"2023-03-25T01:48:56.988086+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T01:48:45.924614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T01:48:45.924614+0000","last_clean_scrub_stamp":"2023-03-25T01:48:45.924614+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:34:02.770586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1],"acting":[5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":10,"reported_epoch":19,"state":"active+clean","last_fresh":"2023-03-25T01:48:58.909220+0000","last_change":"2023-03-25T01:48:56.990419+0000","last_active":"2023-03-25T01:48:58.909220+0000","last_peered":"2023-03-25T01:48:58.909220+0000","last_clean":"2023-03-25T01:48:58.909220+0000","last_became_active":"2023-03-25T01:48:56.990188+0000","last_became_peered":"2023-03-25T01:48:56.990188+0000","last_unstale":"2023-03-25T01:48:58.909220+0000","last_undegraded":"2023-03-25T01:48:58.909220+0000","last_fullsized":"2023-03-25T01:48:58.909220+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T01:48:55.982031+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T01:48:55.982031+0000","last_clean_scrub_stamp":"2023-03-25T01:48:55.982031+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:43:07.120795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":14,"reported_epoch":19,"state":"active+clean","last_fresh":"2023-03-25T01:48:57.402752+0000","last_change":"2023-03-25T01:48:46.942191+0000","last_active":"2023-03-25T01:48:57.402752+0000","last_peered":"2023-03-25T01:48:57.402752+0000","last_clean":"2023-03-25T01:48:57.402752+0000","last_became_active":"2023-03-25T01:48:46.941918+0000","last_became_peered":"2023-03-25T01:48:46.941918+0000","last_unstale":"2023-03-25T01:48:57.402752+0000","last_undegraded":"2023-03-25T01:48:57.402752+0000","last_fullsized":"2023-03-25T01:48:57.402752+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T01:48:45.924614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T01:48:45.924614+0000","last_clean_scrub_stamp":"2023-03-25T01:48:45.924614+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T10:36:32.544700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5],"acting":[2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":10,"reported_epoch":20,"state":"active+clean","last_fresh":"2023-03-25T01:48:57.985996+0000","last_change":"2023-03-25T01:48:57.404656+0000","last_active":"2023-03-25T01:48:57.985996+0000","last_peered":"2023-03-25T01:48:57.985996+0000","last_clean":"2023-03-25T01:48:57.985996+0000","last_became_active":"2023-03-25T01:48:57.404416+0000","last_became_peered":"2023-03-25T01:48:57.404416+0000","last_unstale":"2023-03-25T01:48:57.985996+0000","last_undegraded":"2023-03-25T01:48:57.985996+0000","last_fullsized":"2023-03-25T01:48:57.985996+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T01:48:55.982031+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T01:48:55.982031+0000","last_clean_scrub_stamp":"2023-03-25T01:48:55.982031+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T12:21:35.179737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":15,"reported_epoch":19,"state":"active+clean","last_fresh":"2023-03-25T01:48:56.987983+0000","last_change":"2023-03-25T01:48:47.283824+0000","last_active":"2023-03-25T01:48:56.987983+0000","last_peered":"2023-03-25T01:48:56.987983+0000","last_clean":"2023-03-25T01:48:56.987983+0000","last_became_active":"2023-03-25T01:48:47.283534+0000","last_became_peered":"2023-03-25T01:48:47.283534+0000","last_unstale":"2023-03-25T01:48:56.987983+0000","last_undegraded":"2023-03-25T01:48:56.987983+0000","last_fullsized":"2023-03-25T01:48:56.987983+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T01:48:45.924614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T01:48:45.924614+0000","last_clean_scrub_stamp":"2023-03-25T01:48:45.924614+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T10:36:32.544700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4],"acting":[5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"13'97","reported_seq":142,"reported_epoch":19,"state":"active+clean","last_fresh":"2023-03-25T01:48:56.988118+0000","last_change":"2023-03-25T01:48:45.934080+0000","last_active":"2023-03-25T01:48:56.988118+0000","last_peered":"2023-03-25T01:48:56.988118+0000","last_clean":"2023-03-25T01:48:56.988118+0000","last_became_active":"2023-03-25T01:48:45.933746+0000","last_became_peered":"2023-03-25T01:48:45.933746+0000","last_unstale":"2023-03-25T01:48:56.988118+0000","last_undegraded":"2023-03-25T01:48:56.988118+0000","last_fullsized":"2023-03-25T01:48:56.988118+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T01:48:44.920729+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T01:48:44.920729+0000","last_clean_scrub_stamp":"2023-03-25T01:48:44.920729+0000","objects_scrubbed":0,"log_size":97,"log_dups_size":0,"ondisk_log_size":97,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:43:52.243010+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":137,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.2","version":"20'30","reported_seq":40,"reported_epoch":20,"state":"active+clean","last_fresh":"2023-03-25T01:49:06.431894+0000","last_change":"2023-03-25T01:48:56.991149+0000","last_active":"2023-03-25T01:49:06.431894+0000","last_peered":"2023-03-25T01:49:06.431894+0000","last_clean":"2023-03-25T01:49:06.431894+0000","last_became_active":"2023-03-25T01:48:56.990889+0000","last_became_peered":"2023-03-25T01:48:56.990889+0000","last_unstale":"2023-03-25T01:49:06.431894+0000","last_undegraded":"2023-03-25T01:49:06.431894+0000","last_fullsized":"2023-03-25T01:49:06.431894+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T01:48:55.982031+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T01:48:55.982031+0000","last_clean_scrub_stamp":"2023-03-25T01:48:55.982031+0000","objects_scrubbed":0,"log_size":30,"log_dups_size":0,"ondisk_log_size":30,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T10:51:06.108055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":650000,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":30,"num_write_kb":649,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5],"acting":[3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":14,"reported_epoch":19,"state":"active+clean","last_fresh":"2023-03-25T01:48:56.984507+0000","last_change":"2023-03-25T01:48:46.943089+0000","last_active":"2023-03-25T01:48:56.984507+0000","last_peered":"2023-03-25T01:48:56.984507+0000","last_clean":"2023-03-25T01:48:56.984507+0000","last_became_active":"2023-03-25T01:48:46.942843+0000","last_became_peered":"2023-03-25T01:48:46.942843+0000","last_unstale":"2023-03-25T01:48:56.984507+0000","last_undegraded":"2023-03-25T01:48:56.984507+0000","last_fullsized":"2023-03-25T01:48:56.984507+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T01:48:45.924614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T01:48:45.924614+0000","last_clean_scrub_stamp":"2023-03-25T01:48:45.924614+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T07:21:10.392042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5],"acting":[3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1","version":"20'34","reported_seq":44,"reported_epoch":20,"state":"active+clean","last_fresh":"2023-03-25T01:49:13.951565+0000","last_change":"2023-03-25T01:48:56.989334+0000","last_active":"2023-03-25T01:49:13.951565+0000","last_peered":"2023-03-25T01:49:13.951565+0000","last_clean":"2023-03-25T01:49:13.951565+0000","last_became_active":"2023-03-25T01:48:56.989079+0000","last_became_peered":"2023-03-25T01:48:56.989079+0000","last_unstale":"2023-03-25T01:49:13.951565+0000","last_undegraded":"2023-03-25T01:49:13.951565+0000","last_fullsized":"2023-03-25T01:49:13.951565+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T01:48:55.982031+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T01:48:55.982031+0000","last_clean_scrub_stamp":"2023-03-25T01:48:55.982031+0000","objects_scrubbed":0,"log_size":34,"log_dups_size":0,"ondisk_log_size":34,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T07:51:55.253730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":330000,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":34,"num_write_kb":339,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":14,"reported_epoch":19,"state":"active+clean","last_fresh":"2023-03-25T01:48:56.984518+0000","last_change":"2023-03-25T01:48:47.283615+0000","last_active":"2023-03-25T01:48:56.984518+0000","last_peered":"2023-03-25T01:48:56.984518+0000","last_clean":"2023-03-25T01:48:56.984518+0000","last_became_active":"2023-03-25T01:48:47.283359+0000","last_became_peered":"2023-03-25T01:48:47.283359+0000","last_unstale":"2023-03-25T01:48:56.984518+0000","last_undegraded":"2023-03-25T01:48:56.984518+0000","last_fullsized":"2023-03-25T01:48:56.984518+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T01:48:45.924614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T01:48:45.924614+0000","last_clean_scrub_stamp":"2023-03-25T01:48:45.924614+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T02:55:58.247062+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.4","version":"20'5","reported_seq":15,"reported_epoch":20,"state":"active+clean","last_fresh":"2023-03-25T01:48:58.920001+0000","last_change":"2023-03-25T01:48:57.404723+0000","last_active":"2023-03-25T01:48:58.920001+0000","last_peered":"2023-03-25T01:48:58.920001+0000","last_clean":"2023-03-25T01:48:58.920001+0000","last_became_active":"2023-03-25T01:48:57.404409+0000","last_became_peered":"2023-03-25T01:48:57.404409+0000","last_unstale":"2023-03-25T01:48:58.920001+0000","last_undegraded":"2023-03-25T01:48:58.920001+0000","last_fullsized":"2023-03-25T01:48:58.920001+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T01:48:55.982031+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T01:48:55.982031+0000","last_clean_scrub_stamp":"2023-03-25T01:48:55.982031+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T07:51:55.253730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":640000,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5,"num_write_kb":627,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"20'88","reported_seq":98,"reported_epoch":20,"state":"active+clean","last_fresh":"2023-03-25T01:49:18.814701+0000","last_change":"2023-03-25T01:48:56.988823+0000","last_active":"2023-03-25T01:49:18.814701+0000","last_peered":"2023-03-25T01:49:18.814701+0000","last_clean":"2023-03-25T01:49:18.814701+0000","last_became_active":"2023-03-25T01:48:56.988568+0000","last_became_peered":"2023-03-25T01:48:56.988568+0000","last_unstale":"2023-03-25T01:49:18.814701+0000","last_undegraded":"2023-03-25T01:49:18.814701+0000","last_fullsized":"2023-03-25T01:49:18.814701+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T01:48:55.982031+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T01:48:55.982031+0000","last_clean_scrub_stamp":"2023-03-25T01:48:55.982031+0000","objects_scrubbed":0,"log_size":88,"log_dups_size":0,"ondisk_log_size":88,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T10:51:06.108055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1310000,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":88,"num_write_kb":1314,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":14,"reported_epoch":19,"state":"active+clean","last_fresh":"2023-03-25T01:48:56.988499+0000","last_change":"2023-03-25T01:48:47.347152+0000","last_active":"2023-03-25T01:48:56.988499+0000","last_peered":"2023-03-25T01:48:56.988499+0000","last_clean":"2023-03-25T01:48:56.988499+0000","last_became_active":"2023-03-25T01:48:47.346919+0000","last_became_peered":"2023-03-25T01:48:47.346919+0000","last_unstale":"2023-03-25T01:48:56.988499+0000","last_undegraded":"2023-03-25T01:48:56.988499+0000","last_fullsized":"2023-03-25T01:48:56.988499+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T01:48:45.924614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T01:48:45.924614+0000","last_clean_scrub_stamp":"2023-03-25T01:48:45.924614+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T02:55:58.247062+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":14,"reported_epoch":19,"state":"active+clean","last_fresh":"2023-03-25T01:48:56.984528+0000","last_change":"2023-03-25T01:48:47.347110+0000","last_active":"2023-03-25T01:48:56.984528+0000","last_peered":"2023-03-25T01:48:56.984528+0000","last_clean":"2023-03-25T01:48:56.984528+0000","last_became_active":"2023-03-25T01:48:47.346832+0000","last_became_peered":"2023-03-25T01:48:47.346832+0000","last_unstale":"2023-03-25T01:48:56.984528+0000","last_undegraded":"2023-03-25T01:48:56.984528+0000","last_fullsized":"2023-03-25T01:48:56.984528+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T01:48:45.924614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T01:48:45.924614+0000","last_clean_scrub_stamp":"2023-03-25T01:48:45.924614+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:34:02.770586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"20'126","reported_seq":136,"reported_epoch":20,"state":"active+clean","last_fresh":"2023-03-25T01:49:17.626037+0000","last_change":"2023-03-25T01:48:56.990341+0000","last_active":"2023-03-25T01:49:17.626037+0000","last_peered":"2023-03-25T01:49:17.626037+0000","last_clean":"2023-03-25T01:49:17.626037+0000","last_became_active":"2023-03-25T01:48:56.990039+0000","last_became_peered":"2023-03-25T01:48:56.990039+0000","last_unstale":"2023-03-25T01:49:17.626037+0000","last_undegraded":"2023-03-25T01:49:17.626037+0000","last_fullsized":"2023-03-25T01:49:17.626037+0000","mapping_epoch":18,"log_start":"0'0","ondisk_log_start":"0'0","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T01:48:55.982031+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T01:48:55.982031+0000","last_clean_scrub_stamp":"2023-03-25T01:48:55.982031+0000","objects_scrubbed":0,"log_size":126,"log_dups_size":0,"ondisk_log_size":126,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T06:14:10.792416+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1630000,"num_objects":5,"num_object_clones":0,"num_object_copies":10,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":126,"num_write_kb":1653,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":5,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":3,"num_pg":12,"stat_sum":{"num_bytes":6510000,"num_objects":20,"num_object_clones":0,"num_object_copies":40,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":20,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":403,"num_write_kb":6544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":19,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":13123584,"data_stored":13020000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":403,"ondisk_log_size":403,"up":24,"acting":24,"num_store_stats":6},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":137,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":97,"ondisk_log_size":97,"up":2,"acting":2,"num_store_stats":2},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":6}],"osd_stats":[{"osd":5,"up_from":11,"seq":47244640265,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":104857600,"kb_used":6608,"kb_used_data":1480,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":104850992,"statfs":{"total":107374182400,"available":107367415808,"internally_reserved":0,"allocated":1515520,"data_stored":1366284,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,3,4],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34999999999999998}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.307}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29399999999999998}]},{"osd":3,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.434}]},{"osd":4,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.36499999999999999}]}]},{"osd":4,"up_from":11,"seq":47244640267,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":104857600,"kb_used":8024,"kb_used_data":2768,"kb_used_omap":0,"kb_used_meta":5248,"kb_avail":104849576,"statfs":{"total":107374182400,"available":107365965824,"internally_reserved":0,"allocated":2834432,"data_stored":2676284,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5373952},"hb_peers":[0,1,2,3,5],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.31900000000000001}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.24099999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.25800000000000001}]},{"osd":3,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.41499999999999998}]},{"osd":5,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.435}]}]},{"osd":3,"up_from":11,"seq":47244640266,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":104857600,"kb_used":8736,"kb_used_data":3416,"kb_used_omap":0,"kb_used_meta":5312,"kb_avail":104848864,"statfs":{"total":107374182400,"available":107365236736,"internally_reserved":0,"allocated":3497984,"data_stored":3336284,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5439488},"hb_peers":[0,1,2,4,5],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.30199999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.35599999999999998}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.38400000000000001}]},{"osd":4,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.312}]},{"osd":5,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.32000000000000001}]}]},{"osd":2,"up_from":11,"seq":47244640266,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":104857600,"kb_used":6916,"kb_used_data":1788,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":104850684,"statfs":{"total":107374182400,"available":107367100416,"internally_reserved":0,"allocated":1830912,"data_stored":1676284,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,3,4,5],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.498}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.45200000000000001}]},{"osd":3,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.52800000000000002}]},{"osd":4,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.32100000000000001}]},{"osd":5,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.58299999999999996}]}]},{"osd":0,"up_from":11,"seq":47244640265,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":104857600,"kb_used":8348,"kb_used_data":2708,"kb_used_omap":0,"kb_used_meta":5632,"kb_avail":104849252,"statfs":{"total":107374182400,"available":107365634048,"internally_reserved":0,"allocated":2772992,"data_stored":2616652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5767168},"hb_peers":[1,2,3,4,5],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.32800000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26000000000000001}]},{"osd":3,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.30399999999999999}]},{"osd":4,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.24399999999999999}]},{"osd":5,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":1,"up_from":11,"seq":47244640266,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":104857600,"kb_used":8656,"kb_used_data":3016,"kb_used_omap":0,"kb_used_meta":5632,"kb_avail":104848944,"statfs":{"total":107374182400,"available":107365318656,"internally_reserved":0,"allocated":3088384,"data_stored":2926652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5767168},"hb_peers":[0,2,3,4,5],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.27300000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34799999999999998}]},{"osd":3,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.24399999999999999}]},{"osd":4,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.36499999999999999}]},{"osd":5,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.49099999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1974272,"data_stored":1960000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2289664,"data_stored":2270000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1626112,"data_stored":1610000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":3293184,"data_stored":3270000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2629632,"data_stored":2610000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1310720,"data_stored":1300000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T02:15:44.816 INFO:tasks.ceph:Checking cluster log for badness...
, 2023-03-25T02:15:44.817 DEBUG:teuthology.orchestra.run.smithi087:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/ceph.log | egrep -v 'overall HEALTH_' | egrep -v '\(OSDMAP_FLAGS\)' | egrep -v '\(OSD_' | egrep -v '\(PG_' | egrep -v '\(TOO_FEW_PGS\)' | egrep -v '\(POOL_APP_NOT_ENABLED\)' | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1
, 2023-03-25T02:15:44.885 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/ceph-0 on ubuntu@smithi087.front.sepia.ceph.com

Ignoring malparsed backtrace: 2023-03-25T02:15:48.639 DEBUG:teuthology.orchestra.run.smithi087:> sudo tar cz -f - -C /home/ubuntu/cephtest/archive -- .
, 2023-03-25T02:15:57.238 INFO:teuthology.task.internal:Transferring binaries for coredumps...
, 2023-03-25T02:15:57.288 INFO:teuthology.task.internal: core looks like: /home/teuthworker/archive/benhanokh-2023-03-16_20:57:19-rados-WIP_GBH_snap_mapper_5-distro-default-smithi/7210931/remote/smithi087/coredump/1679708966.134597.core: ELF 64-bit LSB core file x86-64, version 1 (SYSV), SVR4-style, from 'ceph-objectstore-tool --data-path /var/lib/ceph/osd/ceph-3 --journal-path /var/', real uid: 0, effective uid: 0, real gid: 0, effective gid: 0, execfn: '/bin/ceph-objectstore-tool', platform: 'x86_64'

Ignoring malparsed backtrace:  ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
,  1: /lib64/libpthread.so.0(+0x12cf0) [0x7f0a88840cf0]
,  2: __pthread_mutex_lock()

Ignoring malparsed backtrace: gzip: /var/log/ceph/valgrind/*.gz: No such file or directory
, gzip: /var/log/ceph/valgrind/*.gz: No such file or directory
, 2023-03-25T13:09:44.107 INFO:tasks.ceph:Archiving crash dumps...

Ignoring malparsed backtrace:     fetch_binaries_for_coredumps(path, rem)
,   File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/task/internal/__init__.py", line 345, in fetch_binaries_for_coredumps
,     remote_path = remote.sh(['which', dump_program]).rstrip()

Ignoring malparsed backtrace:  ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
,  1: /lib64/libpthread.so.0(+0x12cf0) [0x7f0a88840cf0]
,  2: __pthread_mutex_lock()

Ignoring malparsed backtrace: gzip: /var/log/ceph/valgrind/*.gz: No such file or directory
, gzip: /var/log/ceph/valgrind/*.gz: No such file or directory
, 2023-03-25T13:09:44.107 INFO:tasks.ceph:Archiving crash dumps...

Ignoring malparsed backtrace:     fetch_binaries_for_coredumps(path, rem)
,   File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/task/internal/__init__.py", line 345, in fetch_binaries_for_coredumps
,     remote_path = remote.sh(['which', dump_program]).rstrip()

Ignoring malparsed backtrace:  ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
,  1: /lib64/libpthread.so.0(+0x12cf0) [0x7f0a88840cf0]
,  2: __pthread_mutex_lock()

Ignoring malparsed backtrace: gzip: /var/log/ceph/valgrind/*.gz: No such file or directory
, gzip: /var/log/ceph/valgrind/*.gz: No such file or directory
, 2023-03-25T13:09:44.107 INFO:tasks.ceph:Archiving crash dumps...

Ignoring malparsed backtrace:     fetch_binaries_for_coredumps(path, rem)
,   File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/task/internal/__init__.py", line 345, in fetch_binaries_for_coredumps
,     remote_path = remote.sh(['which', dump_program]).rstrip()

Ignoring malparsed backtrace:  ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
,  1: /lib64/libpthread.so.0(+0x12cf0) [0x7f0a88840cf0]
,  2: __pthread_mutex_lock()

Ignoring malparsed backtrace: gzip: /var/log/ceph/valgrind/*.gz: No such file or directory
, gzip: /var/log/ceph/valgrind/*.gz: No such file or directory
, 2023-03-25T13:09:44.107 INFO:tasks.ceph:Archiving crash dumps...

Ignoring malparsed backtrace:     fetch_binaries_for_coredumps(path, rem)
,   File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/task/internal/__init__.py", line 345, in fetch_binaries_for_coredumps
,     remote_path = remote.sh(['which', dump_program]).rstrip()

Ignoring malparsed backtrace:  ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
,  1: /lib64/libpthread.so.0(+0x12cf0) [0x7f8137776cf0]
,  2: __pthread_mutex_lock()

Ignoring malparsed backtrace: 2023-03-25T10:36:28.117 DEBUG:teuthology.orchestra.run.smithi129:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd deep-scrub 1
, instructed osd(s) 1 to deep-scrub
, 2023-03-25T10:36:28.450 INFO:tasks.ceph:Scrubbing osd.2

Ignoring malparsed backtrace: 2023-03-25T10:37:12.609 DEBUG:teuthology.orchestra.run.smithi129:> sudo find /var/log/ceph -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip --
, 2023-03-25T10:37:15.667 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.2 has been restored
, 2023-03-25T10:37:15.667 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.mon.b has been restored

Ignoring malparsed backtrace:  ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
,  1: /lib64/libpthread.so.0(+0x12cf0) [0x7f8137776cf0]
,  2: __pthread_mutex_lock()

Ignoring malparsed backtrace: 2023-03-25T10:36:28.117 DEBUG:teuthology.orchestra.run.smithi129:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd deep-scrub 1
, instructed osd(s) 1 to deep-scrub
, 2023-03-25T10:36:28.450 INFO:tasks.ceph:Scrubbing osd.2

Ignoring malparsed backtrace: 2023-03-25T10:37:12.609 DEBUG:teuthology.orchestra.run.smithi129:> sudo find /var/log/ceph -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip --
, 2023-03-25T10:37:15.667 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.2 has been restored
, 2023-03-25T10:37:15.667 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.mon.b has been restored

Ignoring malparsed backtrace:  ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
,  1: /lib64/libpthread.so.0(+0x12cf0) [0x7f8137776cf0]
,  2: __pthread_mutex_lock()

Ignoring malparsed backtrace: 2023-03-25T10:36:28.117 DEBUG:teuthology.orchestra.run.smithi129:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd deep-scrub 1
, instructed osd(s) 1 to deep-scrub
, 2023-03-25T10:36:28.450 INFO:tasks.ceph:Scrubbing osd.2

Ignoring malparsed backtrace: 2023-03-25T10:37:12.609 DEBUG:teuthology.orchestra.run.smithi129:> sudo find /var/log/ceph -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip --
, 2023-03-25T10:37:15.667 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.2 has been restored
, 2023-03-25T10:37:15.667 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.mon.b has been restored

Ignoring malparsed backtrace:  ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
,  1: /lib64/libpthread.so.0(+0x12cf0) [0x7f8137776cf0]
,  2: __pthread_mutex_lock()

Ignoring malparsed backtrace: 2023-03-25T10:36:28.117 DEBUG:teuthology.orchestra.run.smithi129:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd deep-scrub 1
, instructed osd(s) 1 to deep-scrub
, 2023-03-25T10:36:28.450 INFO:tasks.ceph:Scrubbing osd.2

Ignoring malparsed backtrace: 2023-03-25T10:37:12.609 DEBUG:teuthology.orchestra.run.smithi129:> sudo find /var/log/ceph -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip --
, 2023-03-25T10:37:15.667 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.2 has been restored
, 2023-03-25T10:37:15.667 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.mon.b has been restored

Ignoring malparsed backtrace:  ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
,  1: /lib64/libpthread.so.0(+0x12cf0) [0x7f8137776cf0]
,  2: __pthread_mutex_lock()

Ignoring malparsed backtrace: 2023-03-25T10:36:28.117 DEBUG:teuthology.orchestra.run.smithi129:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd deep-scrub 1
, instructed osd(s) 1 to deep-scrub
, 2023-03-25T10:36:28.450 INFO:tasks.ceph:Scrubbing osd.2

Ignoring malparsed backtrace: 2023-03-25T10:37:12.609 DEBUG:teuthology.orchestra.run.smithi129:> sudo find /var/log/ceph -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip --
, 2023-03-25T10:37:15.667 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.2 has been restored
, 2023-03-25T10:37:15.667 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.mon.b has been restored

Ignoring malparsed backtrace:  ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
,  1: /lib64/libpthread.so.0(+0x12cf0) [0x7f8137776cf0]
,  2: __pthread_mutex_lock()

Ignoring malparsed backtrace: 2023-03-25T10:36:28.117 DEBUG:teuthology.orchestra.run.smithi129:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd deep-scrub 1
, instructed osd(s) 1 to deep-scrub
, 2023-03-25T10:36:28.450 INFO:tasks.ceph:Scrubbing osd.2

Ignoring malparsed backtrace: 2023-03-25T10:37:12.609 DEBUG:teuthology.orchestra.run.smithi129:> sudo find /var/log/ceph -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip --
, 2023-03-25T10:37:15.667 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.2 has been restored
, 2023-03-25T10:37:15.667 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.mon.b has been restored

Ignoring malparsed backtrace:  ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
,  1: /lib64/libpthread.so.0(+0x12ce0) [0x7f3593b91ce0]
,  2: __pthread_mutex_lock()

Ignoring malparsed backtrace: /home/ubuntu/cephtest/clone.client.0/qa/standalone/ceph-helpers.sh:200: teardown:  '[' yes = yes ']'
, 2023-03-25T13:44:25.968 INFO:tasks.workunit.client.0.smithi184.stdout:ERROR: Failure due to cores found
, /home/ubuntu/cephtest/clone.client.0/qa/standalone/ceph-helpers.sh:201: teardown:  echo 'ERROR: Failure due to cores found'

Ignoring malparsed backtrace:     r.wait()
,   File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/orchestra/run.py", line 161, in wait
,     self._raise_for_status()

Ignoring malparsed backtrace:  ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
,  1: /lib64/libpthread.so.0(+0x12ce0) [0x7f3593b91ce0]
,  2: __pthread_mutex_lock()

Ignoring malparsed backtrace: /home/ubuntu/cephtest/clone.client.0/qa/standalone/ceph-helpers.sh:200: teardown:  '[' yes = yes ']'
, 2023-03-25T13:44:25.968 INFO:tasks.workunit.client.0.smithi184.stdout:ERROR: Failure due to cores found
, /home/ubuntu/cephtest/clone.client.0/qa/standalone/ceph-helpers.sh:201: teardown:  echo 'ERROR: Failure due to cores found'

Ignoring malparsed backtrace:     r.wait()
,   File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/orchestra/run.py", line 161, in wait
,     self._raise_for_status()

Ignoring malparsed backtrace:  ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
,  1: /lib64/libpthread.so.0(+0x12ce0) [0x7f3593b91ce0]
,  2: __pthread_mutex_lock()

Ignoring malparsed backtrace: /home/ubuntu/cephtest/clone.client.0/qa/standalone/ceph-helpers.sh:200: teardown:  '[' yes = yes ']'
, 2023-03-25T13:44:25.968 INFO:tasks.workunit.client.0.smithi184.stdout:ERROR: Failure due to cores found
, /home/ubuntu/cephtest/clone.client.0/qa/standalone/ceph-helpers.sh:201: teardown:  echo 'ERROR: Failure due to cores found'

Ignoring malparsed backtrace:     r.wait()
,   File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/orchestra/run.py", line 161, in wait
,     self._raise_for_status()

Ignoring malparsed backtrace:  ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
,  1: /lib64/libpthread.so.0(+0x12ce0) [0x7f3593b91ce0]
,  2: __pthread_mutex_lock()

Ignoring malparsed backtrace: /home/ubuntu/cephtest/clone.client.0/qa/standalone/ceph-helpers.sh:200: teardown:  '[' yes = yes ']'
, 2023-03-25T13:44:25.968 INFO:tasks.workunit.client.0.smithi184.stdout:ERROR: Failure due to cores found
, /home/ubuntu/cephtest/clone.client.0/qa/standalone/ceph-helpers.sh:201: teardown:  echo 'ERROR: Failure due to cores found'

Ignoring malparsed backtrace:     r.wait()
,   File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/orchestra/run.py", line 161, in wait
,     self._raise_for_status()

Ignoring malparsed backtrace:  ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
,  1: /lib64/libpthread.so.0(+0x12ce0) [0x7f3593b91ce0]
,  2: __pthread_mutex_lock()

Ignoring malparsed backtrace: /home/ubuntu/cephtest/clone.client.0/qa/standalone/ceph-helpers.sh:200: teardown:  '[' yes = yes ']'
, 2023-03-25T13:44:25.968 INFO:tasks.workunit.client.0.smithi184.stdout:ERROR: Failure due to cores found
, /home/ubuntu/cephtest/clone.client.0/qa/standalone/ceph-helpers.sh:201: teardown:  echo 'ERROR: Failure due to cores found'

Ignoring malparsed backtrace:     r.wait()
,   File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/orchestra/run.py", line 161, in wait
,     self._raise_for_status()

Ignoring malparsed backtrace:  ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
,  1: /lib64/libpthread.so.0(+0x12ce0) [0x7f3593b91ce0]
,  2: __pthread_mutex_lock()

Ignoring malparsed backtrace: /home/ubuntu/cephtest/clone.client.0/qa/standalone/ceph-helpers.sh:200: teardown:  '[' yes = yes ']'
, 2023-03-25T13:44:25.968 INFO:tasks.workunit.client.0.smithi184.stdout:ERROR: Failure due to cores found
, /home/ubuntu/cephtest/clone.client.0/qa/standalone/ceph-helpers.sh:201: teardown:  echo 'ERROR: Failure due to cores found'

Ignoring malparsed backtrace:     r.wait()
,   File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/orchestra/run.py", line 161, in wait
,     self._raise_for_status()

Ignoring malparsed backtrace:  ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
,  1: /lib64/libpthread.so.0(+0x12ce0) [0x7f3593b91ce0]
,  2: __pthread_mutex_lock()

Ignoring malparsed backtrace: /home/ubuntu/cephtest/clone.client.0/qa/standalone/ceph-helpers.sh:200: teardown:  '[' yes = yes ']'
, 2023-03-25T13:44:25.968 INFO:tasks.workunit.client.0.smithi184.stdout:ERROR: Failure due to cores found
, /home/ubuntu/cephtest/clone.client.0/qa/standalone/ceph-helpers.sh:201: teardown:  echo 'ERROR: Failure due to cores found'

Ignoring malparsed backtrace:     r.wait()
,   File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/orchestra/run.py", line 161, in wait
,     self._raise_for_status()

Ignoring malparsed backtrace:  ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
,  1: /lib64/libpthread.so.0(+0x12ce0) [0x7f3593b91ce0]
,  2: __pthread_mutex_lock()

Ignoring malparsed backtrace: /home/ubuntu/cephtest/clone.client.0/qa/standalone/ceph-helpers.sh:200: teardown:  '[' yes = yes ']'
, 2023-03-25T13:44:25.968 INFO:tasks.workunit.client.0.smithi184.stdout:ERROR: Failure due to cores found
, /home/ubuntu/cephtest/clone.client.0/qa/standalone/ceph-helpers.sh:201: teardown:  echo 'ERROR: Failure due to cores found'

Ignoring malparsed backtrace:     r.wait()
,   File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/orchestra/run.py", line 161, in wait
,     self._raise_for_status()

Ignoring malparsed backtrace:  ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
,  1: /lib64/libpthread.so.0(+0x12ce0) [0x7f9275707ce0]
,  2: __pthread_mutex_lock()

Ignoring malparsed backtrace: 2023-03-25T11:09:01.783 INFO:tasks.workunit.client.0.smithi177.stdout:> mwatts:x:1138:
, 2023-03-25T11:09:01.783 INFO:tasks.workunit.client.0.smithi177.stdout:> prsrivas:x:1139:
, 2023-03-25T11:09:01.783 INFO:tasks.workunit.client.0.smithi177.stdout:> pdonnell:x:1140:

Ignoring malparsed backtrace:  ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
,  1: /lib64/libpthread.so.0(+0x12ce0) [0x7ffb9236dce0]
,  2: __pthread_mutex_lock()

Ignoring malparsed backtrace:   File "/home/teuthworker/src/github.com_ceph_ceph-c_511015c23eef0684a64171b710e432448df2b7c7/qa/tasks/workunit.py", line 295, in _spawn_on_all_clients
,     p.spawn(_run_tests, ctx, refspec, role, [unit], env,
,   File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/parallel.py", line 84, in __exit__

Ignoring malparsed backtrace: /usr/bin/which: no 0 in (./td/ceph-helpers)
, /usr/bin/which: no NOTEXISTS in (/home/ubuntu/.local/bin:/home/ubuntu/bin:/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin)
, 2023-03-25T11:09:07.088 INFO:teuthology.orchestra.run.smithi177.stdout:/usr/bin/ceph-objectstore-tool

Ignoring malparsed backtrace:  ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
,  1: /lib64/libpthread.so.0(+0x12ce0) [0x7f9275707ce0]
,  2: __pthread_mutex_lock()

Ignoring malparsed backtrace: 2023-03-25T11:09:01.783 INFO:tasks.workunit.client.0.smithi177.stdout:> mwatts:x:1138:
, 2023-03-25T11:09:01.783 INFO:tasks.workunit.client.0.smithi177.stdout:> prsrivas:x:1139:
, 2023-03-25T11:09:01.783 INFO:tasks.workunit.client.0.smithi177.stdout:> pdonnell:x:1140:

Ignoring malparsed backtrace:  ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
,  1: /lib64/libpthread.so.0(+0x12ce0) [0x7ffb9236dce0]
,  2: __pthread_mutex_lock()

Ignoring malparsed backtrace:   File "/home/teuthworker/src/github.com_ceph_ceph-c_511015c23eef0684a64171b710e432448df2b7c7/qa/tasks/workunit.py", line 295, in _spawn_on_all_clients
,     p.spawn(_run_tests, ctx, refspec, role, [unit], env,
,   File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/parallel.py", line 84, in __exit__

Ignoring malparsed backtrace: /usr/bin/which: no 0 in (./td/ceph-helpers)
, /usr/bin/which: no NOTEXISTS in (/home/ubuntu/.local/bin:/home/ubuntu/bin:/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin)
, 2023-03-25T11:09:07.088 INFO:teuthology.orchestra.run.smithi177.stdout:/usr/bin/ceph-objectstore-tool

Ignoring malparsed backtrace:  ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
,  1: /lib64/libpthread.so.0(+0x12ce0) [0x7f9275707ce0]
,  2: __pthread_mutex_lock()

Ignoring malparsed backtrace: 2023-03-25T11:09:01.783 INFO:tasks.workunit.client.0.smithi177.stdout:> mwatts:x:1138:
, 2023-03-25T11:09:01.783 INFO:tasks.workunit.client.0.smithi177.stdout:> prsrivas:x:1139:
, 2023-03-25T11:09:01.783 INFO:tasks.workunit.client.0.smithi177.stdout:> pdonnell:x:1140:

Ignoring malparsed backtrace:  ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
,  1: /lib64/libpthread.so.0(+0x12ce0) [0x7ffb9236dce0]
,  2: __pthread_mutex_lock()

Ignoring malparsed backtrace:   File "/home/teuthworker/src/github.com_ceph_ceph-c_511015c23eef0684a64171b710e432448df2b7c7/qa/tasks/workunit.py", line 295, in _spawn_on_all_clients
,     p.spawn(_run_tests, ctx, refspec, role, [unit], env,
,   File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/parallel.py", line 84, in __exit__

Ignoring malparsed backtrace: /usr/bin/which: no 0 in (./td/ceph-helpers)
, /usr/bin/which: no NOTEXISTS in (/home/ubuntu/.local/bin:/home/ubuntu/bin:/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin)
, 2023-03-25T11:09:07.088 INFO:teuthology.orchestra.run.smithi177.stdout:/usr/bin/ceph-objectstore-tool

Ignoring malparsed backtrace:  ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
,  1: /lib64/libpthread.so.0(+0x12ce0) [0x7f9275707ce0]
,  2: __pthread_mutex_lock()

Ignoring malparsed backtrace: 2023-03-25T11:09:01.783 INFO:tasks.workunit.client.0.smithi177.stdout:> mwatts:x:1138:
, 2023-03-25T11:09:01.783 INFO:tasks.workunit.client.0.smithi177.stdout:> prsrivas:x:1139:
, 2023-03-25T11:09:01.783 INFO:tasks.workunit.client.0.smithi177.stdout:> pdonnell:x:1140:

Ignoring malparsed backtrace:  ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
,  1: /lib64/libpthread.so.0(+0x12ce0) [0x7ffb9236dce0]
,  2: __pthread_mutex_lock()

Ignoring malparsed backtrace:   File "/home/teuthworker/src/github.com_ceph_ceph-c_511015c23eef0684a64171b710e432448df2b7c7/qa/tasks/workunit.py", line 295, in _spawn_on_all_clients
,     p.spawn(_run_tests, ctx, refspec, role, [unit], env,
,   File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/parallel.py", line 84, in __exit__

Ignoring malparsed backtrace: /usr/bin/which: no 0 in (./td/ceph-helpers)
, /usr/bin/which: no NOTEXISTS in (/home/ubuntu/.local/bin:/home/ubuntu/bin:/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin)
, 2023-03-25T11:09:07.088 INFO:teuthology.orchestra.run.smithi177.stdout:/usr/bin/ceph-objectstore-tool

Ignoring malparsed backtrace:  ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
,  1: /lib64/libpthread.so.0(+0x12ce0) [0x7f9275707ce0]
,  2: __pthread_mutex_lock()

Ignoring malparsed backtrace: 2023-03-25T11:09:01.783 INFO:tasks.workunit.client.0.smithi177.stdout:> mwatts:x:1138:
, 2023-03-25T11:09:01.783 INFO:tasks.workunit.client.0.smithi177.stdout:> prsrivas:x:1139:
, 2023-03-25T11:09:01.783 INFO:tasks.workunit.client.0.smithi177.stdout:> pdonnell:x:1140:

Ignoring malparsed backtrace:  ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
,  1: /lib64/libpthread.so.0(+0x12ce0) [0x7ffb9236dce0]
,  2: __pthread_mutex_lock()

Ignoring malparsed backtrace:   File "/home/teuthworker/src/github.com_ceph_ceph-c_511015c23eef0684a64171b710e432448df2b7c7/qa/tasks/workunit.py", line 295, in _spawn_on_all_clients
,     p.spawn(_run_tests, ctx, refspec, role, [unit], env,
,   File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/parallel.py", line 84, in __exit__

Ignoring malparsed backtrace: /usr/bin/which: no 0 in (./td/ceph-helpers)
, /usr/bin/which: no NOTEXISTS in (/home/ubuntu/.local/bin:/home/ubuntu/bin:/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin)
, 2023-03-25T11:09:07.088 INFO:teuthology.orchestra.run.smithi177.stdout:/usr/bin/ceph-objectstore-tool

Ignoring malparsed backtrace:  ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
,  1: /lib64/libpthread.so.0(+0x12ce0) [0x7f9275707ce0]
,  2: __pthread_mutex_lock()

Ignoring malparsed backtrace: 2023-03-25T11:09:01.783 INFO:tasks.workunit.client.0.smithi177.stdout:> mwatts:x:1138:
, 2023-03-25T11:09:01.783 INFO:tasks.workunit.client.0.smithi177.stdout:> prsrivas:x:1139:
, 2023-03-25T11:09:01.783 INFO:tasks.workunit.client.0.smithi177.stdout:> pdonnell:x:1140:

Ignoring malparsed backtrace:  ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
,  1: /lib64/libpthread.so.0(+0x12ce0) [0x7ffb9236dce0]
,  2: __pthread_mutex_lock()

Ignoring malparsed backtrace:   File "/home/teuthworker/src/github.com_ceph_ceph-c_511015c23eef0684a64171b710e432448df2b7c7/qa/tasks/workunit.py", line 295, in _spawn_on_all_clients
,     p.spawn(_run_tests, ctx, refspec, role, [unit], env,
,   File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/parallel.py", line 84, in __exit__

Ignoring malparsed backtrace: /usr/bin/which: no 0 in (./td/ceph-helpers)
, /usr/bin/which: no NOTEXISTS in (/home/ubuntu/.local/bin:/home/ubuntu/bin:/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin)
, 2023-03-25T11:09:07.088 INFO:teuthology.orchestra.run.smithi177.stdout:/usr/bin/ceph-objectstore-tool

Ignoring malparsed backtrace:  ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
,  1: /lib64/libpthread.so.0(+0x12ce0) [0x7f9275707ce0]
,  2: __pthread_mutex_lock()

Ignoring malparsed backtrace: 2023-03-25T11:09:01.783 INFO:tasks.workunit.client.0.smithi177.stdout:> mwatts:x:1138:
, 2023-03-25T11:09:01.783 INFO:tasks.workunit.client.0.smithi177.stdout:> prsrivas:x:1139:
, 2023-03-25T11:09:01.783 INFO:tasks.workunit.client.0.smithi177.stdout:> pdonnell:x:1140:

Ignoring malparsed backtrace:  ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
,  1: /lib64/libpthread.so.0(+0x12ce0) [0x7ffb9236dce0]
,  2: __pthread_mutex_lock()

Ignoring malparsed backtrace:   File "/home/teuthworker/src/github.com_ceph_ceph-c_511015c23eef0684a64171b710e432448df2b7c7/qa/tasks/workunit.py", line 295, in _spawn_on_all_clients
,     p.spawn(_run_tests, ctx, refspec, role, [unit], env,
,   File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/parallel.py", line 84, in __exit__

Ignoring malparsed backtrace: /usr/bin/which: no 0 in (./td/ceph-helpers)
, /usr/bin/which: no NOTEXISTS in (/home/ubuntu/.local/bin:/home/ubuntu/bin:/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin)
, 2023-03-25T11:09:07.088 INFO:teuthology.orchestra.run.smithi177.stdout:/usr/bin/ceph-objectstore-tool

Ignoring malparsed backtrace:  ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
,  1: /lib64/libpthread.so.0(+0x12ce0) [0x7f9275707ce0]
,  2: __pthread_mutex_lock()

Ignoring malparsed backtrace: 2023-03-25T11:09:01.783 INFO:tasks.workunit.client.0.smithi177.stdout:> mwatts:x:1138:
, 2023-03-25T11:09:01.783 INFO:tasks.workunit.client.0.smithi177.stdout:> prsrivas:x:1139:
, 2023-03-25T11:09:01.783 INFO:tasks.workunit.client.0.smithi177.stdout:> pdonnell:x:1140:

Ignoring malparsed backtrace:  ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
,  1: /lib64/libpthread.so.0(+0x12ce0) [0x7ffb9236dce0]
,  2: __pthread_mutex_lock()

Ignoring malparsed backtrace:   File "/home/teuthworker/src/github.com_ceph_ceph-c_511015c23eef0684a64171b710e432448df2b7c7/qa/tasks/workunit.py", line 295, in _spawn_on_all_clients
,     p.spawn(_run_tests, ctx, refspec, role, [unit], env,
,   File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/parallel.py", line 84, in __exit__

Ignoring malparsed backtrace: /usr/bin/which: no 0 in (./td/ceph-helpers)
, /usr/bin/which: no NOTEXISTS in (/home/ubuntu/.local/bin:/home/ubuntu/bin:/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin)
, 2023-03-25T11:09:07.088 INFO:teuthology.orchestra.run.smithi177.stdout:/usr/bin/ceph-objectstore-tool

Ignoring malparsed backtrace:  ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
,  1: /lib64/libpthread.so.0(+0x12ce0) [0x7f9275707ce0]
,  2: __pthread_mutex_lock()

Ignoring malparsed backtrace: 2023-03-25T11:09:01.783 INFO:tasks.workunit.client.0.smithi177.stdout:> mwatts:x:1138:
, 2023-03-25T11:09:01.783 INFO:tasks.workunit.client.0.smithi177.stdout:> prsrivas:x:1139:
, 2023-03-25T11:09:01.783 INFO:tasks.workunit.client.0.smithi177.stdout:> pdonnell:x:1140:

Ignoring malparsed backtrace:  ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
,  1: /lib64/libpthread.so.0(+0x12ce0) [0x7ffb9236dce0]
,  2: __pthread_mutex_lock()

Ignoring malparsed backtrace:   File "/home/teuthworker/src/github.com_ceph_ceph-c_511015c23eef0684a64171b710e432448df2b7c7/qa/tasks/workunit.py", line 295, in _spawn_on_all_clients
,     p.spawn(_run_tests, ctx, refspec, role, [unit], env,
,   File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/parallel.py", line 84, in __exit__

Ignoring malparsed backtrace: /usr/bin/which: no 0 in (./td/ceph-helpers)
, /usr/bin/which: no NOTEXISTS in (/home/ubuntu/.local/bin:/home/ubuntu/bin:/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin)
, 2023-03-25T11:09:07.088 INFO:teuthology.orchestra.run.smithi177.stdout:/usr/bin/ceph-objectstore-tool

Ignoring malparsed backtrace:  ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
,  1: /lib/x86_64-linux-gnu/libpthread.so.0(+0x14420) [0x7fc83a246420]
,  2: pthread_mutex_lock()

Ignoring malparsed backtrace: 2023-03-25T13:08:33.248 DEBUG:teuthology.orchestra.run.smithi186:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd deep-scrub 1
, instructed osd(s) 1 to deep-scrub
, 2023-03-25T13:08:33.558 INFO:tasks.ceph:Scrubbing osd.2

Ignoring malparsed backtrace: 2023-03-25T13:09:10.535 DEBUG:teuthology.orchestra.run.smithi186:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true
, 2023-03-25T13:09:10.557 INFO:teuthology.orchestra.run.smithi186.stdout:     remote           refid      st t when poll reach   delay   offset  jitter
, 2023-03-25T13:09:10.557 INFO:teuthology.orchestra.run.smithi186.stdout:==============================================================================

Ignoring malparsed backtrace:  ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
,  1: /lib/x86_64-linux-gnu/libpthread.so.0(+0x14420) [0x7fc83a246420]
,  2: pthread_mutex_lock()

Ignoring malparsed backtrace: 2023-03-25T13:08:33.248 DEBUG:teuthology.orchestra.run.smithi186:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd deep-scrub 1
, instructed osd(s) 1 to deep-scrub
, 2023-03-25T13:08:33.558 INFO:tasks.ceph:Scrubbing osd.2

Ignoring malparsed backtrace: 2023-03-25T13:09:10.535 DEBUG:teuthology.orchestra.run.smithi186:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true
, 2023-03-25T13:09:10.557 INFO:teuthology.orchestra.run.smithi186.stdout:     remote           refid      st t when poll reach   delay   offset  jitter
, 2023-03-25T13:09:10.557 INFO:teuthology.orchestra.run.smithi186.stdout:==============================================================================

Ignoring malparsed backtrace:  ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
,  1: /lib/x86_64-linux-gnu/libpthread.so.0(+0x14420) [0x7fc83a246420]
,  2: pthread_mutex_lock()

Ignoring malparsed backtrace: 2023-03-25T13:08:33.248 DEBUG:teuthology.orchestra.run.smithi186:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd deep-scrub 1
, instructed osd(s) 1 to deep-scrub
, 2023-03-25T13:08:33.558 INFO:tasks.ceph:Scrubbing osd.2

Ignoring malparsed backtrace: 2023-03-25T13:09:10.535 DEBUG:teuthology.orchestra.run.smithi186:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true
, 2023-03-25T13:09:10.557 INFO:teuthology.orchestra.run.smithi186.stdout:     remote           refid      st t when poll reach   delay   offset  jitter
, 2023-03-25T13:09:10.557 INFO:teuthology.orchestra.run.smithi186.stdout:==============================================================================

Ignoring malparsed backtrace:  ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
,  1: /lib/x86_64-linux-gnu/libpthread.so.0(+0x14420) [0x7fc83a246420]
,  2: pthread_mutex_lock()

Ignoring malparsed backtrace: 2023-03-25T13:08:33.248 DEBUG:teuthology.orchestra.run.smithi186:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd deep-scrub 1
, instructed osd(s) 1 to deep-scrub
, 2023-03-25T13:08:33.558 INFO:tasks.ceph:Scrubbing osd.2

Ignoring malparsed backtrace: 2023-03-25T13:09:10.535 DEBUG:teuthology.orchestra.run.smithi186:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true
, 2023-03-25T13:09:10.557 INFO:teuthology.orchestra.run.smithi186.stdout:     remote           refid      st t when poll reach   delay   offset  jitter
, 2023-03-25T13:09:10.557 INFO:teuthology.orchestra.run.smithi186.stdout:==============================================================================

Ignoring malparsed backtrace:  ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
,  1: /lib/x86_64-linux-gnu/libpthread.so.0(+0x14420) [0x7fc6e94be420]
,  2: pthread_mutex_lock()

Ignoring malparsed backtrace: gzip: /var/log/ceph/valgrind/*.gz: No such file or directory
, gzip: /var/log/ceph/valgrind/*.gz: No such file or directory
, 2023-03-24T21:33:23.912 INFO:tasks.ceph:Archiving crash dumps...

Ignoring malparsed backtrace: 2023-03-24T21:33:29.484 INFO:teuthology.task.internal:Not uploading archives.
, 2023-03-24T21:33:29.485 DEBUG:teuthology.run_tasks:Unwinding manager internal.base
, 2023-03-24T21:33:29.495 INFO:teuthology.task.internal:Tidying up after the test...

Ignoring malparsed backtrace:  ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
,  1: /lib/x86_64-linux-gnu/libpthread.so.0(+0x14420) [0x7fc6e94be420]
,  2: pthread_mutex_lock()

Ignoring malparsed backtrace: gzip: /var/log/ceph/valgrind/*.gz: No such file or directory
, gzip: /var/log/ceph/valgrind/*.gz: No such file or directory
, 2023-03-24T21:33:23.912 INFO:tasks.ceph:Archiving crash dumps...

Ignoring malparsed backtrace: 2023-03-24T21:33:29.484 INFO:teuthology.task.internal:Not uploading archives.
, 2023-03-24T21:33:29.485 DEBUG:teuthology.run_tasks:Unwinding manager internal.base
, 2023-03-24T21:33:29.495 INFO:teuthology.task.internal:Tidying up after the test...

Ignoring malparsed backtrace:  ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
,  1: /lib64/libpthread.so.0(+0x12ce0) [0x7f3cbe08dce0]
,  2: __pthread_mutex_lock()

Ignoring malparsed backtrace: 2023-03-24T21:39:59.186+0000 7fe4827dc700 -1 osd.1 15 heartbeat_check: no reply from 172.21.15.44:6812 osd.0 since back 2023-03-24T21:38:54.008087+0000 front 2023-03-24T21:38:54.008074+0000 (oldest deadline 2023-03-24T21:39:17.508138+0000)
, 2023-03-24T21:39:59.367+0000 7fe1e46e2700 -1 osd.2 15 heartbeat_check: no reply from 172.21.15.44:6812 osd.0 since back 2023-03-24T21:38:51.721293+0000 front 2023-03-24T21:38:51.721319+0000 (oldest deadline 2023-03-24T21:39:17.621390+0000)
, 2023-03-24T21:39:59.367+0000 7fe1e46e2700 -1 osd.2 15 get_health_metrics reporting 1 slow ops, oldest is osd_op(client.4179.0:4 1.1 1:88c1567c:::rbd_trash_purge_schedule:head [omap-get-vals in=16b] snapc 0=[] ondisk+read+known_if_redirected+supports_pool_eio e14)

Ignoring malparsed backtrace: 2023-03-24T21:40:32.178+0000 7fe1e46e2700 -1 osd.2 15 get_health_metrics reporting 1 slow ops, oldest is osd_op(client.4179.0:4 1.1 1:88c1567c:::rbd_trash_purge_schedule:head [omap-get-vals in=16b] snapc 0=[] ondisk+read+known_if_redirected+supports_pool_eio e14)
, 2023-03-24T21:40:33.131+0000 7fe1e46e2700 -1 osd.2 15 heartbeat_check: no reply from 172.21.15.44:6812 osd.0 since back 2023-03-24T21:38:51.721293+0000 front 2023-03-24T21:38:51.721319+0000 (oldest deadline 2023-03-24T21:39:17.621390+0000)
, 2023-03-24T21:40:33.131+0000 7fe1e46e2700 -1 osd.2 15 get_health_metrics reporting 1 slow ops, oldest is osd_op(client.4179.0:4 1.1 1:88c1567c:::rbd_trash_purge_schedule:head [omap-get-vals in=16b] snapc 0=[] ondisk+read+known_if_redirected+supports_pool_eio e14)

Ignoring malparsed backtrace: 2023-03-24T21:40:55.921+0000 7fe1e46e2700 -1 osd.2 15 heartbeat_check: no reply from 172.21.15.44:6812 osd.0 since back 2023-03-24T21:38:51.721293+0000 front 2023-03-24T21:38:51.721319+0000 (oldest deadline 2023-03-24T21:39:17.621390+0000)
, 2023-03-24T21:40:55.921+0000 7fe1e46e2700 -1 osd.2 15 get_health_metrics reporting 1 slow ops, oldest is osd_op(client.4179.0:4 1.1 1:88c1567c:::rbd_trash_purge_schedule:head [omap-get-vals in=16b] snapc 0=[] ondisk+read+known_if_redirected+supports_pool_eio e14)
, 2023-03-24T21:40:55.929 INFO:tasks.ceph.osd.1:Stopped

Ignoring malparsed backtrace:   File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/orchestra/remote.py", line 642, in get_tar
,     self.run(args=args)
,   File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/orchestra/remote.py", line 525, in run

Ignoring malparsed backtrace: 2023-03-24T21:42:19.352 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog
, 2023-03-24T21:42:19.424 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring...
, 2023-03-24T21:42:19.426 DEBUG:teuthology.orchestra.run.smithi044:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart

Ignoring malparsed backtrace: During handling of the above exception, another exception occurred:
, 
, Traceback (most recent call last):

Ignoring malparsed backtrace:  ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
,  1: /lib64/libpthread.so.0(+0x12ce0) [0x7f3cbe08dce0]
,  2: __pthread_mutex_lock()

Ignoring malparsed backtrace: 2023-03-24T21:39:59.186+0000 7fe4827dc700 -1 osd.1 15 heartbeat_check: no reply from 172.21.15.44:6812 osd.0 since back 2023-03-24T21:38:54.008087+0000 front 2023-03-24T21:38:54.008074+0000 (oldest deadline 2023-03-24T21:39:17.508138+0000)
, 2023-03-24T21:39:59.367+0000 7fe1e46e2700 -1 osd.2 15 heartbeat_check: no reply from 172.21.15.44:6812 osd.0 since back 2023-03-24T21:38:51.721293+0000 front 2023-03-24T21:38:51.721319+0000 (oldest deadline 2023-03-24T21:39:17.621390+0000)
, 2023-03-24T21:39:59.367+0000 7fe1e46e2700 -1 osd.2 15 get_health_metrics reporting 1 slow ops, oldest is osd_op(client.4179.0:4 1.1 1:88c1567c:::rbd_trash_purge_schedule:head [omap-get-vals in=16b] snapc 0=[] ondisk+read+known_if_redirected+supports_pool_eio e14)

Ignoring malparsed backtrace: 2023-03-24T21:40:32.178+0000 7fe1e46e2700 -1 osd.2 15 get_health_metrics reporting 1 slow ops, oldest is osd_op(client.4179.0:4 1.1 1:88c1567c:::rbd_trash_purge_schedule:head [omap-get-vals in=16b] snapc 0=[] ondisk+read+known_if_redirected+supports_pool_eio e14)
, 2023-03-24T21:40:33.131+0000 7fe1e46e2700 -1 osd.2 15 heartbeat_check: no reply from 172.21.15.44:6812 osd.0 since back 2023-03-24T21:38:51.721293+0000 front 2023-03-24T21:38:51.721319+0000 (oldest deadline 2023-03-24T21:39:17.621390+0000)
, 2023-03-24T21:40:33.131+0000 7fe1e46e2700 -1 osd.2 15 get_health_metrics reporting 1 slow ops, oldest is osd_op(client.4179.0:4 1.1 1:88c1567c:::rbd_trash_purge_schedule:head [omap-get-vals in=16b] snapc 0=[] ondisk+read+known_if_redirected+supports_pool_eio e14)

Ignoring malparsed backtrace: 2023-03-24T21:40:55.921+0000 7fe1e46e2700 -1 osd.2 15 heartbeat_check: no reply from 172.21.15.44:6812 osd.0 since back 2023-03-24T21:38:51.721293+0000 front 2023-03-24T21:38:51.721319+0000 (oldest deadline 2023-03-24T21:39:17.621390+0000)
, 2023-03-24T21:40:55.921+0000 7fe1e46e2700 -1 osd.2 15 get_health_metrics reporting 1 slow ops, oldest is osd_op(client.4179.0:4 1.1 1:88c1567c:::rbd_trash_purge_schedule:head [omap-get-vals in=16b] snapc 0=[] ondisk+read+known_if_redirected+supports_pool_eio e14)
, 2023-03-24T21:40:55.929 INFO:tasks.ceph.osd.1:Stopped

Ignoring malparsed backtrace:   File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/orchestra/remote.py", line 642, in get_tar
,     self.run(args=args)
,   File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/orchestra/remote.py", line 525, in run

Ignoring malparsed backtrace: 2023-03-24T21:42:19.352 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog
, 2023-03-24T21:42:19.424 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring...
, 2023-03-24T21:42:19.426 DEBUG:teuthology.orchestra.run.smithi044:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart

Ignoring malparsed backtrace: During handling of the above exception, another exception occurred:
, 
, Traceback (most recent call last):

Ignoring malparsed backtrace:  ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
,  1: /lib64/libpthread.so.0(+0x12ce0) [0x7f22b503dce0]
,  2: __pthread_mutex_lock()

Ignoring malparsed backtrace:     raise exc.exc_info[1]
,   File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/parallel.py", line 23, in capture_traceback
,     return func(*args, **kwargs)

Ignoring malparsed backtrace:  ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
,  1: /lib64/libpthread.so.0(+0x12ce0) [0x7f22b503dce0]
,  2: __pthread_mutex_lock()

Ignoring malparsed backtrace:     raise exc.exc_info[1]
,   File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/parallel.py", line 23, in capture_traceback
,     return func(*args, **kwargs)

Ignoring malparsed backtrace:  ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
,  1: /lib64/libpthread.so.0(+0x12ce0) [0x7f22b503dce0]
,  2: __pthread_mutex_lock()

Ignoring malparsed backtrace:     raise exc.exc_info[1]
,   File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/parallel.py", line 23, in capture_traceback
,     return func(*args, **kwargs)

Ignoring malparsed backtrace:  ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
,  1: /lib64/libpthread.so.0(+0x12ce0) [0x7f22b503dce0]
,  2: __pthread_mutex_lock()

Ignoring malparsed backtrace:     raise exc.exc_info[1]
,   File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/parallel.py", line 23, in capture_traceback
,     return func(*args, **kwargs)

Ignoring malparsed backtrace:  ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
,  1: /lib64/libpthread.so.0(+0x12ce0) [0x7f22b503dce0]
,  2: __pthread_mutex_lock()

Ignoring malparsed backtrace:     raise exc.exc_info[1]
,   File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/parallel.py", line 23, in capture_traceback
,     return func(*args, **kwargs)

Ignoring malparsed backtrace:  ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
,  1: /lib64/libpthread.so.0(+0x12ce0) [0x7f22b503dce0]
,  2: __pthread_mutex_lock()

Ignoring malparsed backtrace:     raise exc.exc_info[1]
,   File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/parallel.py", line 23, in capture_traceback
,     return func(*args, **kwargs)

Ignoring malparsed backtrace:  ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
,  1: /lib64/libpthread.so.0(+0x12ce0) [0x7f22b503dce0]
,  2: __pthread_mutex_lock()

Ignoring malparsed backtrace:     raise exc.exc_info[1]
,   File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/parallel.py", line 23, in capture_traceback
,     return func(*args, **kwargs)

Ignoring malparsed backtrace:  ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
,  1: /lib64/libpthread.so.0(+0x12ce0) [0x7f22b503dce0]
,  2: __pthread_mutex_lock()

Ignoring malparsed backtrace:     raise exc.exc_info[1]
,   File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/parallel.py", line 23, in capture_traceback
,     return func(*args, **kwargs)

Ignoring malparsed backtrace:  ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
,  1: /lib64/libpthread.so.0(+0x12ce0) [0x7f22b503dce0]
,  2: __pthread_mutex_lock()

Ignoring malparsed backtrace:     raise exc.exc_info[1]
,   File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/parallel.py", line 23, in capture_traceback
,     return func(*args, **kwargs)

Ignoring malparsed backtrace:  ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
,  1: /lib64/libpthread.so.0(+0x12ce0) [0x7f22b503dce0]
,  2: __pthread_mutex_lock()

Ignoring malparsed backtrace:     raise exc.exc_info[1]
,   File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/parallel.py", line 23, in capture_traceback
,     return func(*args, **kwargs)

Ignoring malparsed backtrace:  ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
,  1: /lib64/libpthread.so.0(+0x12ce0) [0x7f22b503dce0]
,  2: __pthread_mutex_lock()

Ignoring malparsed backtrace:     raise exc.exc_info[1]
,   File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/parallel.py", line 23, in capture_traceback
,     return func(*args, **kwargs)

Ignoring malparsed backtrace:  ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
,  1: /lib64/libpthread.so.0(+0x12ce0) [0x7f22b503dce0]
,  2: __pthread_mutex_lock()

Ignoring malparsed backtrace:     raise exc.exc_info[1]
,   File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/parallel.py", line 23, in capture_traceback
,     return func(*args, **kwargs)

Ignoring malparsed backtrace:  ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
,  1: /lib64/libpthread.so.0(+0x12ce0) [0x7fe187790ce0]
,  2: __pthread_mutex_lock()

Ignoring malparsed backtrace: 2023-03-25T06:33:44.245 INFO:tasks.ceph.osd.3:Stopped
, 2023-03-25T06:33:44.245 INFO:teuthology.misc:Shutting down mgr daemons...
, 2023-03-25T06:33:44.246 DEBUG:tasks.ceph.mgr.x:waiting for process to exit

Ignoring malparsed backtrace: 2023-03-25T06:33:47.811 WARNING:teuthology.task.internal:Found coredumps on ubuntu@smithi070.front.sepia.ceph.com, flagging run as failed
, 2023-03-25T06:33:47.812 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive
, 2023-03-25T06:33:47.840 INFO:teuthology.task.internal:Transferring archived files...

Ignoring malparsed backtrace:  ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
,  1: /lib64/libpthread.so.0(+0x12ce0) [0x7fe187790ce0]
,  2: __pthread_mutex_lock()

Ignoring malparsed backtrace: 2023-03-25T06:33:44.245 INFO:tasks.ceph.osd.3:Stopped
, 2023-03-25T06:33:44.245 INFO:teuthology.misc:Shutting down mgr daemons...
, 2023-03-25T06:33:44.246 DEBUG:tasks.ceph.mgr.x:waiting for process to exit

Ignoring malparsed backtrace: 2023-03-25T06:33:47.811 WARNING:teuthology.task.internal:Found coredumps on ubuntu@smithi070.front.sepia.ceph.com, flagging run as failed
, 2023-03-25T06:33:47.812 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive
, 2023-03-25T06:33:47.840 INFO:teuthology.task.internal:Transferring archived files...

Ignoring malparsed backtrace:  ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
,  1: /lib64/libpthread.so.0(+0x12ce0) [0x7f62f6f51ce0]
,  2: __pthread_mutex_lock()

Ignoring malparsed backtrace: 2023-03-24T22:36:52.171 DEBUG:teuthology.orchestra.run.smithi115:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd deep-scrub 1
, instructed osd(s) 1 to deep-scrub
, 2023-03-24T22:36:52.503 INFO:tasks.ceph:Scrubbing osd.2

Ignoring malparsed backtrace: 2023-03-24T22:37:30.719 DEBUG:teuthology.orchestra.run.smithi115:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true
, bash: ntpq: command not found
, 2023-03-24T22:37:30.738 INFO:teuthology.orchestra.run.smithi115.stdout:MS Name/IP address         Stratum Poll Reach LastRx Last sample

Ignoring malparsed backtrace:  ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
,  1: /lib64/libpthread.so.0(+0x12ce0) [0x7f62f6f51ce0]
,  2: __pthread_mutex_lock()

Ignoring malparsed backtrace: 2023-03-24T22:36:52.171 DEBUG:teuthology.orchestra.run.smithi115:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd deep-scrub 1
, instructed osd(s) 1 to deep-scrub
, 2023-03-24T22:36:52.503 INFO:tasks.ceph:Scrubbing osd.2

Ignoring malparsed backtrace: 2023-03-24T22:37:30.719 DEBUG:teuthology.orchestra.run.smithi115:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true
, bash: ntpq: command not found
, 2023-03-24T22:37:30.738 INFO:teuthology.orchestra.run.smithi115.stdout:MS Name/IP address         Stratum Poll Reach LastRx Last sample

Ignoring malparsed backtrace:  ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
,  1: /lib64/libpthread.so.0(+0x12ce0) [0x7f62f6f51ce0]
,  2: __pthread_mutex_lock()

Ignoring malparsed backtrace: 2023-03-24T22:36:52.171 DEBUG:teuthology.orchestra.run.smithi115:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd deep-scrub 1
, instructed osd(s) 1 to deep-scrub
, 2023-03-24T22:36:52.503 INFO:tasks.ceph:Scrubbing osd.2

Ignoring malparsed backtrace: 2023-03-24T22:37:30.719 DEBUG:teuthology.orchestra.run.smithi115:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true
, bash: ntpq: command not found
, 2023-03-24T22:37:30.738 INFO:teuthology.orchestra.run.smithi115.stdout:MS Name/IP address         Stratum Poll Reach LastRx Last sample

Ignoring malparsed backtrace:  ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
,  1: /lib64/libpthread.so.0(+0x12ce0) [0x7f62f6f51ce0]
,  2: __pthread_mutex_lock()

Ignoring malparsed backtrace: 2023-03-24T22:36:52.171 DEBUG:teuthology.orchestra.run.smithi115:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd deep-scrub 1
, instructed osd(s) 1 to deep-scrub
, 2023-03-24T22:36:52.503 INFO:tasks.ceph:Scrubbing osd.2

Ignoring malparsed backtrace: 2023-03-24T22:37:30.719 DEBUG:teuthology.orchestra.run.smithi115:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true
, bash: ntpq: command not found
, 2023-03-24T22:37:30.738 INFO:teuthology.orchestra.run.smithi115.stdout:MS Name/IP address         Stratum Poll Reach LastRx Last sample

Ignoring malparsed backtrace:  ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
,  1: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x128) [0x55b39fbb1a4b]
,  2: ceph-osd(+0xee9c07) [0x55b39fbb1c07]

Ignoring malparsed backtrace:  ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
,  1: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x128) [0x55b39fbb1a4b]
,  2: ceph-osd(+0xee9c07) [0x55b39fbb1c07]

Ignoring malparsed backtrace:  ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
,  1: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x128) [0x55b39fbb1a4b]
,  2: ceph-osd(+0xee9c07) [0x55b39fbb1c07]

Ignoring malparsed backtrace:  ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
,  1: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x128) [0x55b39fbb1a4b]
,  2: ceph-osd(+0xee9c07) [0x55b39fbb1c07]

Ignoring malparsed backtrace:  ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
,  1: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x128) [0x55b39fbb1a4b]
,  2: ceph-osd(+0xee9c07) [0x55b39fbb1c07]

Ignoring malparsed backtrace:  ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
,  1: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x128) [0x55b39fbb1a4b]
,  2: ceph-osd(+0xee9c07) [0x55b39fbb1c07]

Ignoring malparsed backtrace:  ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
,  1: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x128) [0x55b39fbb1a4b]
,  2: ceph-osd(+0xee9c07) [0x55b39fbb1c07]

Ignoring malparsed backtrace:  ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
,  1: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x128) [0x55b39fbb1a4b]
,  2: ceph-osd(+0xee9c07) [0x55b39fbb1c07]

Ignoring malparsed backtrace:  ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
,  1: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x128) [0x55b39fbb1a4b]
,  2: ceph-osd(+0xee9c07) [0x55b39fbb1c07]

Ignoring malparsed backtrace:  ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
,  1: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x128) [0x55b39fbb1a4b]
,  2: ceph-osd(+0xee9c07) [0x55b39fbb1c07]

Misunderstood line: 
Ignoring malparsed backtrace:  ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
,  1: /lib/x86_64-linux-gnu/libpthread.so.0(+0x14420) [0x7fe43e6fe420]
,  2: pthread_mutex_lock()

Ignoring malparsed backtrace: 2023-03-24T21:56:18.804 INFO:tasks.workunit.client.0.smithi117.stdout:{
, 2023-03-24T21:56:18.805 INFO:tasks.workunit.client.0.smithi117.stdout:    "ops": [],
, 2023-03-24T21:56:18.805 INFO:tasks.workunit.client.0.smithi117.stdout:    "num_ops": 0

Ignoring malparsed backtrace: 2023-03-24T21:56:20.430 INFO:tasks.workunit.client.0.smithi117.stdout:success
, /home/ubuntu/cephtest/clone.client.0/qa/standalone/osd/divergent-priors.sh:619: TEST_divergent_2:  rm td/divergent-priors/td/divergent-priors/exp.30989.out
, rm: cannot remove 'td/divergent-priors/td/divergent-priors/exp.30989.out': No such file or directory

Ignoring malparsed backtrace:   File "/home/teuthworker/src/github.com_ceph_ceph-c_511015c23eef0684a64171b710e432448df2b7c7/qa/tasks/workunit.py", line 145, in task
,     _spawn_on_all_clients(ctx, refspec, all_tasks, config.get('env'),
,   File "/home/teuthworker/src/github.com_ceph_ceph-c_511015c23eef0684a64171b710e432448df2b7c7/qa/tasks/workunit.py", line 295, in _spawn_on_all_clients

Ignoring malparsed backtrace:     suppress = manager.__exit__(*exc_info)
,   File "/usr/lib/python3.8/contextlib.py", line 120, in __exit__
,     next(self.gen)

Ignoring malparsed backtrace:  ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
,  1: /lib/x86_64-linux-gnu/libpthread.so.0(+0x14420) [0x7fe43e6fe420]
,  2: pthread_mutex_lock()

Ignoring malparsed backtrace: 2023-03-24T21:56:18.804 INFO:tasks.workunit.client.0.smithi117.stdout:{
, 2023-03-24T21:56:18.805 INFO:tasks.workunit.client.0.smithi117.stdout:    "ops": [],
, 2023-03-24T21:56:18.805 INFO:tasks.workunit.client.0.smithi117.stdout:    "num_ops": 0

Ignoring malparsed backtrace: 2023-03-24T21:56:20.430 INFO:tasks.workunit.client.0.smithi117.stdout:success
, /home/ubuntu/cephtest/clone.client.0/qa/standalone/osd/divergent-priors.sh:619: TEST_divergent_2:  rm td/divergent-priors/td/divergent-priors/exp.30989.out
, rm: cannot remove 'td/divergent-priors/td/divergent-priors/exp.30989.out': No such file or directory

Ignoring malparsed backtrace:   File "/home/teuthworker/src/github.com_ceph_ceph-c_511015c23eef0684a64171b710e432448df2b7c7/qa/tasks/workunit.py", line 145, in task
,     _spawn_on_all_clients(ctx, refspec, all_tasks, config.get('env'),
,   File "/home/teuthworker/src/github.com_ceph_ceph-c_511015c23eef0684a64171b710e432448df2b7c7/qa/tasks/workunit.py", line 295, in _spawn_on_all_clients

Ignoring malparsed backtrace:     suppress = manager.__exit__(*exc_info)
,   File "/usr/lib/python3.8/contextlib.py", line 120, in __exit__
,     next(self.gen)

Ignoring malparsed backtrace:  ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
,  1: /lib/x86_64-linux-gnu/libpthread.so.0(+0x14420) [0x7fe43e6fe420]
,  2: pthread_mutex_lock()

Ignoring malparsed backtrace: 2023-03-24T21:56:18.804 INFO:tasks.workunit.client.0.smithi117.stdout:{
, 2023-03-24T21:56:18.805 INFO:tasks.workunit.client.0.smithi117.stdout:    "ops": [],
, 2023-03-24T21:56:18.805 INFO:tasks.workunit.client.0.smithi117.stdout:    "num_ops": 0

Ignoring malparsed backtrace: 2023-03-24T21:56:20.430 INFO:tasks.workunit.client.0.smithi117.stdout:success
, /home/ubuntu/cephtest/clone.client.0/qa/standalone/osd/divergent-priors.sh:619: TEST_divergent_2:  rm td/divergent-priors/td/divergent-priors/exp.30989.out
, rm: cannot remove 'td/divergent-priors/td/divergent-priors/exp.30989.out': No such file or directory

Ignoring malparsed backtrace:   File "/home/teuthworker/src/github.com_ceph_ceph-c_511015c23eef0684a64171b710e432448df2b7c7/qa/tasks/workunit.py", line 145, in task
,     _spawn_on_all_clients(ctx, refspec, all_tasks, config.get('env'),
,   File "/home/teuthworker/src/github.com_ceph_ceph-c_511015c23eef0684a64171b710e432448df2b7c7/qa/tasks/workunit.py", line 295, in _spawn_on_all_clients

Ignoring malparsed backtrace:     suppress = manager.__exit__(*exc_info)
,   File "/usr/lib/python3.8/contextlib.py", line 120, in __exit__
,     next(self.gen)

Ignoring malparsed backtrace:  ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
,  1: /lib/x86_64-linux-gnu/libpthread.so.0(+0x14420) [0x7fe43e6fe420]
,  2: pthread_mutex_lock()

Ignoring malparsed backtrace: 2023-03-24T21:56:18.804 INFO:tasks.workunit.client.0.smithi117.stdout:{
, 2023-03-24T21:56:18.805 INFO:tasks.workunit.client.0.smithi117.stdout:    "ops": [],
, 2023-03-24T21:56:18.805 INFO:tasks.workunit.client.0.smithi117.stdout:    "num_ops": 0

Ignoring malparsed backtrace: 2023-03-24T21:56:20.430 INFO:tasks.workunit.client.0.smithi117.stdout:success
, /home/ubuntu/cephtest/clone.client.0/qa/standalone/osd/divergent-priors.sh:619: TEST_divergent_2:  rm td/divergent-priors/td/divergent-priors/exp.30989.out
, rm: cannot remove 'td/divergent-priors/td/divergent-priors/exp.30989.out': No such file or directory

Ignoring malparsed backtrace:   File "/home/teuthworker/src/github.com_ceph_ceph-c_511015c23eef0684a64171b710e432448df2b7c7/qa/tasks/workunit.py", line 145, in task
,     _spawn_on_all_clients(ctx, refspec, all_tasks, config.get('env'),
,   File "/home/teuthworker/src/github.com_ceph_ceph-c_511015c23eef0684a64171b710e432448df2b7c7/qa/tasks/workunit.py", line 295, in _spawn_on_all_clients

Ignoring malparsed backtrace:     suppress = manager.__exit__(*exc_info)
,   File "/usr/lib/python3.8/contextlib.py", line 120, in __exit__
,     next(self.gen)

Ignoring malparsed backtrace:  ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
,  1: /lib/x86_64-linux-gnu/libpthread.so.0(+0x14420) [0x7fe43e6fe420]
,  2: pthread_mutex_lock()

Ignoring malparsed backtrace: 2023-03-24T21:56:18.804 INFO:tasks.workunit.client.0.smithi117.stdout:{
, 2023-03-24T21:56:18.805 INFO:tasks.workunit.client.0.smithi117.stdout:    "ops": [],
, 2023-03-24T21:56:18.805 INFO:tasks.workunit.client.0.smithi117.stdout:    "num_ops": 0

Ignoring malparsed backtrace: 2023-03-24T21:56:20.430 INFO:tasks.workunit.client.0.smithi117.stdout:success
, /home/ubuntu/cephtest/clone.client.0/qa/standalone/osd/divergent-priors.sh:619: TEST_divergent_2:  rm td/divergent-priors/td/divergent-priors/exp.30989.out
, rm: cannot remove 'td/divergent-priors/td/divergent-priors/exp.30989.out': No such file or directory

Ignoring malparsed backtrace:   File "/home/teuthworker/src/github.com_ceph_ceph-c_511015c23eef0684a64171b710e432448df2b7c7/qa/tasks/workunit.py", line 145, in task
,     _spawn_on_all_clients(ctx, refspec, all_tasks, config.get('env'),
,   File "/home/teuthworker/src/github.com_ceph_ceph-c_511015c23eef0684a64171b710e432448df2b7c7/qa/tasks/workunit.py", line 295, in _spawn_on_all_clients

Ignoring malparsed backtrace:     suppress = manager.__exit__(*exc_info)
,   File "/usr/lib/python3.8/contextlib.py", line 120, in __exit__
,     next(self.gen)

Ignoring malparsed backtrace:  ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
,  1: /lib/x86_64-linux-gnu/libpthread.so.0(+0x14420) [0x7fe43e6fe420]
,  2: pthread_mutex_lock()

Ignoring malparsed backtrace: 2023-03-24T21:56:18.804 INFO:tasks.workunit.client.0.smithi117.stdout:{
, 2023-03-24T21:56:18.805 INFO:tasks.workunit.client.0.smithi117.stdout:    "ops": [],
, 2023-03-24T21:56:18.805 INFO:tasks.workunit.client.0.smithi117.stdout:    "num_ops": 0

Ignoring malparsed backtrace: 2023-03-24T21:56:20.430 INFO:tasks.workunit.client.0.smithi117.stdout:success
, /home/ubuntu/cephtest/clone.client.0/qa/standalone/osd/divergent-priors.sh:619: TEST_divergent_2:  rm td/divergent-priors/td/divergent-priors/exp.30989.out
, rm: cannot remove 'td/divergent-priors/td/divergent-priors/exp.30989.out': No such file or directory

Ignoring malparsed backtrace:   File "/home/teuthworker/src/github.com_ceph_ceph-c_511015c23eef0684a64171b710e432448df2b7c7/qa/tasks/workunit.py", line 145, in task
,     _spawn_on_all_clients(ctx, refspec, all_tasks, config.get('env'),
,   File "/home/teuthworker/src/github.com_ceph_ceph-c_511015c23eef0684a64171b710e432448df2b7c7/qa/tasks/workunit.py", line 295, in _spawn_on_all_clients

Ignoring malparsed backtrace:     suppress = manager.__exit__(*exc_info)
,   File "/usr/lib/python3.8/contextlib.py", line 120, in __exit__
,     next(self.gen)

Ignoring malparsed backtrace:  ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
,  1: /lib/x86_64-linux-gnu/libpthread.so.0(+0x14420) [0x7fe43e6fe420]
,  2: pthread_mutex_lock()

Ignoring malparsed backtrace: 2023-03-24T21:56:18.804 INFO:tasks.workunit.client.0.smithi117.stdout:{
, 2023-03-24T21:56:18.805 INFO:tasks.workunit.client.0.smithi117.stdout:    "ops": [],
, 2023-03-24T21:56:18.805 INFO:tasks.workunit.client.0.smithi117.stdout:    "num_ops": 0

Ignoring malparsed backtrace: 2023-03-24T21:56:20.430 INFO:tasks.workunit.client.0.smithi117.stdout:success
, /home/ubuntu/cephtest/clone.client.0/qa/standalone/osd/divergent-priors.sh:619: TEST_divergent_2:  rm td/divergent-priors/td/divergent-priors/exp.30989.out
, rm: cannot remove 'td/divergent-priors/td/divergent-priors/exp.30989.out': No such file or directory

Ignoring malparsed backtrace:   File "/home/teuthworker/src/github.com_ceph_ceph-c_511015c23eef0684a64171b710e432448df2b7c7/qa/tasks/workunit.py", line 145, in task
,     _spawn_on_all_clients(ctx, refspec, all_tasks, config.get('env'),
,   File "/home/teuthworker/src/github.com_ceph_ceph-c_511015c23eef0684a64171b710e432448df2b7c7/qa/tasks/workunit.py", line 295, in _spawn_on_all_clients

Ignoring malparsed backtrace:     suppress = manager.__exit__(*exc_info)
,   File "/usr/lib/python3.8/contextlib.py", line 120, in __exit__
,     next(self.gen)

Ignoring malparsed backtrace:  ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
,  1: /lib/x86_64-linux-gnu/libpthread.so.0(+0x14420) [0x7fe43e6fe420]
,  2: pthread_mutex_lock()

Ignoring malparsed backtrace: 2023-03-24T21:56:18.804 INFO:tasks.workunit.client.0.smithi117.stdout:{
, 2023-03-24T21:56:18.805 INFO:tasks.workunit.client.0.smithi117.stdout:    "ops": [],
, 2023-03-24T21:56:18.805 INFO:tasks.workunit.client.0.smithi117.stdout:    "num_ops": 0

Ignoring malparsed backtrace: 2023-03-24T21:56:20.430 INFO:tasks.workunit.client.0.smithi117.stdout:success
, /home/ubuntu/cephtest/clone.client.0/qa/standalone/osd/divergent-priors.sh:619: TEST_divergent_2:  rm td/divergent-priors/td/divergent-priors/exp.30989.out
, rm: cannot remove 'td/divergent-priors/td/divergent-priors/exp.30989.out': No such file or directory

Ignoring malparsed backtrace:   File "/home/teuthworker/src/github.com_ceph_ceph-c_511015c23eef0684a64171b710e432448df2b7c7/qa/tasks/workunit.py", line 145, in task
,     _spawn_on_all_clients(ctx, refspec, all_tasks, config.get('env'),
,   File "/home/teuthworker/src/github.com_ceph_ceph-c_511015c23eef0684a64171b710e432448df2b7c7/qa/tasks/workunit.py", line 295, in _spawn_on_all_clients

Ignoring malparsed backtrace:     suppress = manager.__exit__(*exc_info)
,   File "/usr/lib/python3.8/contextlib.py", line 120, in __exit__
,     next(self.gen)

Ignoring malparsed backtrace:  ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
,  1: /lib/x86_64-linux-gnu/libpthread.so.0(+0x14420) [0x7fe43e6fe420]
,  2: pthread_mutex_lock()

Ignoring malparsed backtrace: 2023-03-24T21:56:18.804 INFO:tasks.workunit.client.0.smithi117.stdout:{
, 2023-03-24T21:56:18.805 INFO:tasks.workunit.client.0.smithi117.stdout:    "ops": [],
, 2023-03-24T21:56:18.805 INFO:tasks.workunit.client.0.smithi117.stdout:    "num_ops": 0

Ignoring malparsed backtrace: 2023-03-24T21:56:20.430 INFO:tasks.workunit.client.0.smithi117.stdout:success
, /home/ubuntu/cephtest/clone.client.0/qa/standalone/osd/divergent-priors.sh:619: TEST_divergent_2:  rm td/divergent-priors/td/divergent-priors/exp.30989.out
, rm: cannot remove 'td/divergent-priors/td/divergent-priors/exp.30989.out': No such file or directory

Ignoring malparsed backtrace:   File "/home/teuthworker/src/github.com_ceph_ceph-c_511015c23eef0684a64171b710e432448df2b7c7/qa/tasks/workunit.py", line 145, in task
,     _spawn_on_all_clients(ctx, refspec, all_tasks, config.get('env'),
,   File "/home/teuthworker/src/github.com_ceph_ceph-c_511015c23eef0684a64171b710e432448df2b7c7/qa/tasks/workunit.py", line 295, in _spawn_on_all_clients

Ignoring malparsed backtrace:     suppress = manager.__exit__(*exc_info)
,   File "/usr/lib/python3.8/contextlib.py", line 120, in __exit__
,     next(self.gen)

Ignoring malparsed backtrace:  ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
,  1: /lib/x86_64-linux-gnu/libpthread.so.0(+0x14420) [0x7fe43e6fe420]
,  2: pthread_mutex_lock()

Ignoring malparsed backtrace: 2023-03-24T21:56:18.804 INFO:tasks.workunit.client.0.smithi117.stdout:{
, 2023-03-24T21:56:18.805 INFO:tasks.workunit.client.0.smithi117.stdout:    "ops": [],
, 2023-03-24T21:56:18.805 INFO:tasks.workunit.client.0.smithi117.stdout:    "num_ops": 0

Ignoring malparsed backtrace: 2023-03-24T21:56:20.430 INFO:tasks.workunit.client.0.smithi117.stdout:success
, /home/ubuntu/cephtest/clone.client.0/qa/standalone/osd/divergent-priors.sh:619: TEST_divergent_2:  rm td/divergent-priors/td/divergent-priors/exp.30989.out
, rm: cannot remove 'td/divergent-priors/td/divergent-priors/exp.30989.out': No such file or directory

Ignoring malparsed backtrace:   File "/home/teuthworker/src/github.com_ceph_ceph-c_511015c23eef0684a64171b710e432448df2b7c7/qa/tasks/workunit.py", line 145, in task
,     _spawn_on_all_clients(ctx, refspec, all_tasks, config.get('env'),
,   File "/home/teuthworker/src/github.com_ceph_ceph-c_511015c23eef0684a64171b710e432448df2b7c7/qa/tasks/workunit.py", line 295, in _spawn_on_all_clients

Ignoring malparsed backtrace:     suppress = manager.__exit__(*exc_info)
,   File "/usr/lib/python3.8/contextlib.py", line 120, in __exit__
,     next(self.gen)

Ignoring malparsed backtrace:  ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
,  1: /lib/x86_64-linux-gnu/libpthread.so.0(+0x14420) [0x7fe43e6fe420]
,  2: pthread_mutex_lock()

Ignoring malparsed backtrace: 2023-03-24T21:56:18.804 INFO:tasks.workunit.client.0.smithi117.stdout:{
, 2023-03-24T21:56:18.805 INFO:tasks.workunit.client.0.smithi117.stdout:    "ops": [],
, 2023-03-24T21:56:18.805 INFO:tasks.workunit.client.0.smithi117.stdout:    "num_ops": 0

Ignoring malparsed backtrace: 2023-03-24T21:56:20.430 INFO:tasks.workunit.client.0.smithi117.stdout:success
, /home/ubuntu/cephtest/clone.client.0/qa/standalone/osd/divergent-priors.sh:619: TEST_divergent_2:  rm td/divergent-priors/td/divergent-priors/exp.30989.out
, rm: cannot remove 'td/divergent-priors/td/divergent-priors/exp.30989.out': No such file or directory

Ignoring malparsed backtrace:   File "/home/teuthworker/src/github.com_ceph_ceph-c_511015c23eef0684a64171b710e432448df2b7c7/qa/tasks/workunit.py", line 145, in task
,     _spawn_on_all_clients(ctx, refspec, all_tasks, config.get('env'),
,   File "/home/teuthworker/src/github.com_ceph_ceph-c_511015c23eef0684a64171b710e432448df2b7c7/qa/tasks/workunit.py", line 295, in _spawn_on_all_clients

Ignoring malparsed backtrace:     suppress = manager.__exit__(*exc_info)
,   File "/usr/lib/python3.8/contextlib.py", line 120, in __exit__
,     next(self.gen)

Ignoring malparsed backtrace:  ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
,  1: /lib/x86_64-linux-gnu/libpthread.so.0(+0x14420) [0x7fe43e6fe420]
,  2: pthread_mutex_lock()

Ignoring malparsed backtrace: 2023-03-24T21:56:18.804 INFO:tasks.workunit.client.0.smithi117.stdout:{
, 2023-03-24T21:56:18.805 INFO:tasks.workunit.client.0.smithi117.stdout:    "ops": [],
, 2023-03-24T21:56:18.805 INFO:tasks.workunit.client.0.smithi117.stdout:    "num_ops": 0

Ignoring malparsed backtrace: 2023-03-24T21:56:20.430 INFO:tasks.workunit.client.0.smithi117.stdout:success
, /home/ubuntu/cephtest/clone.client.0/qa/standalone/osd/divergent-priors.sh:619: TEST_divergent_2:  rm td/divergent-priors/td/divergent-priors/exp.30989.out
, rm: cannot remove 'td/divergent-priors/td/divergent-priors/exp.30989.out': No such file or directory

Ignoring malparsed backtrace:   File "/home/teuthworker/src/github.com_ceph_ceph-c_511015c23eef0684a64171b710e432448df2b7c7/qa/tasks/workunit.py", line 145, in task
,     _spawn_on_all_clients(ctx, refspec, all_tasks, config.get('env'),
,   File "/home/teuthworker/src/github.com_ceph_ceph-c_511015c23eef0684a64171b710e432448df2b7c7/qa/tasks/workunit.py", line 295, in _spawn_on_all_clients

Ignoring malparsed backtrace:     suppress = manager.__exit__(*exc_info)
,   File "/usr/lib/python3.8/contextlib.py", line 120, in __exit__
,     next(self.gen)

Ignoring malparsed backtrace:  ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
,  1: /lib/x86_64-linux-gnu/libpthread.so.0(+0x14420) [0x7fe43e6fe420]
,  2: pthread_mutex_lock()

Ignoring malparsed backtrace: 2023-03-24T21:56:18.804 INFO:tasks.workunit.client.0.smithi117.stdout:{
, 2023-03-24T21:56:18.805 INFO:tasks.workunit.client.0.smithi117.stdout:    "ops": [],
, 2023-03-24T21:56:18.805 INFO:tasks.workunit.client.0.smithi117.stdout:    "num_ops": 0

Ignoring malparsed backtrace: 2023-03-24T21:56:20.430 INFO:tasks.workunit.client.0.smithi117.stdout:success
, /home/ubuntu/cephtest/clone.client.0/qa/standalone/osd/divergent-priors.sh:619: TEST_divergent_2:  rm td/divergent-priors/td/divergent-priors/exp.30989.out
, rm: cannot remove 'td/divergent-priors/td/divergent-priors/exp.30989.out': No such file or directory

Ignoring malparsed backtrace:   File "/home/teuthworker/src/github.com_ceph_ceph-c_511015c23eef0684a64171b710e432448df2b7c7/qa/tasks/workunit.py", line 145, in task
,     _spawn_on_all_clients(ctx, refspec, all_tasks, config.get('env'),
,   File "/home/teuthworker/src/github.com_ceph_ceph-c_511015c23eef0684a64171b710e432448df2b7c7/qa/tasks/workunit.py", line 295, in _spawn_on_all_clients

Ignoring malparsed backtrace:     suppress = manager.__exit__(*exc_info)
,   File "/usr/lib/python3.8/contextlib.py", line 120, in __exit__
,     next(self.gen)

Ignoring malparsed backtrace:  ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
,  1: /lib/x86_64-linux-gnu/libpthread.so.0(+0x14420) [0x7fe43e6fe420]
,  2: pthread_mutex_lock()

Ignoring malparsed backtrace: 2023-03-24T21:56:18.804 INFO:tasks.workunit.client.0.smithi117.stdout:{
, 2023-03-24T21:56:18.805 INFO:tasks.workunit.client.0.smithi117.stdout:    "ops": [],
, 2023-03-24T21:56:18.805 INFO:tasks.workunit.client.0.smithi117.stdout:    "num_ops": 0

Ignoring malparsed backtrace: 2023-03-24T21:56:20.430 INFO:tasks.workunit.client.0.smithi117.stdout:success
, /home/ubuntu/cephtest/clone.client.0/qa/standalone/osd/divergent-priors.sh:619: TEST_divergent_2:  rm td/divergent-priors/td/divergent-priors/exp.30989.out
, rm: cannot remove 'td/divergent-priors/td/divergent-priors/exp.30989.out': No such file or directory

Ignoring malparsed backtrace:   File "/home/teuthworker/src/github.com_ceph_ceph-c_511015c23eef0684a64171b710e432448df2b7c7/qa/tasks/workunit.py", line 145, in task
,     _spawn_on_all_clients(ctx, refspec, all_tasks, config.get('env'),
,   File "/home/teuthworker/src/github.com_ceph_ceph-c_511015c23eef0684a64171b710e432448df2b7c7/qa/tasks/workunit.py", line 295, in _spawn_on_all_clients

Ignoring malparsed backtrace:     suppress = manager.__exit__(*exc_info)
,   File "/usr/lib/python3.8/contextlib.py", line 120, in __exit__
,     next(self.gen)

Ignoring malparsed backtrace:  ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
,  1: /lib/x86_64-linux-gnu/libpthread.so.0(+0x14420) [0x7fe43e6fe420]
,  2: pthread_mutex_lock()

Ignoring malparsed backtrace: 2023-03-24T21:56:18.804 INFO:tasks.workunit.client.0.smithi117.stdout:{
, 2023-03-24T21:56:18.805 INFO:tasks.workunit.client.0.smithi117.stdout:    "ops": [],
, 2023-03-24T21:56:18.805 INFO:tasks.workunit.client.0.smithi117.stdout:    "num_ops": 0

Ignoring malparsed backtrace: 2023-03-24T21:56:20.430 INFO:tasks.workunit.client.0.smithi117.stdout:success
, /home/ubuntu/cephtest/clone.client.0/qa/standalone/osd/divergent-priors.sh:619: TEST_divergent_2:  rm td/divergent-priors/td/divergent-priors/exp.30989.out
, rm: cannot remove 'td/divergent-priors/td/divergent-priors/exp.30989.out': No such file or directory

Ignoring malparsed backtrace:   File "/home/teuthworker/src/github.com_ceph_ceph-c_511015c23eef0684a64171b710e432448df2b7c7/qa/tasks/workunit.py", line 145, in task
,     _spawn_on_all_clients(ctx, refspec, all_tasks, config.get('env'),
,   File "/home/teuthworker/src/github.com_ceph_ceph-c_511015c23eef0684a64171b710e432448df2b7c7/qa/tasks/workunit.py", line 295, in _spawn_on_all_clients

Ignoring malparsed backtrace:     suppress = manager.__exit__(*exc_info)
,   File "/usr/lib/python3.8/contextlib.py", line 120, in __exit__
,     next(self.gen)

Ignoring malparsed backtrace:  ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
,  1: /lib/x86_64-linux-gnu/libpthread.so.0(+0x14420) [0x7fe43e6fe420]
,  2: pthread_mutex_lock()

Ignoring malparsed backtrace: 2023-03-24T21:56:18.804 INFO:tasks.workunit.client.0.smithi117.stdout:{
, 2023-03-24T21:56:18.805 INFO:tasks.workunit.client.0.smithi117.stdout:    "ops": [],
, 2023-03-24T21:56:18.805 INFO:tasks.workunit.client.0.smithi117.stdout:    "num_ops": 0

Ignoring malparsed backtrace: 2023-03-24T21:56:20.430 INFO:tasks.workunit.client.0.smithi117.stdout:success
, /home/ubuntu/cephtest/clone.client.0/qa/standalone/osd/divergent-priors.sh:619: TEST_divergent_2:  rm td/divergent-priors/td/divergent-priors/exp.30989.out
, rm: cannot remove 'td/divergent-priors/td/divergent-priors/exp.30989.out': No such file or directory

Ignoring malparsed backtrace:   File "/home/teuthworker/src/github.com_ceph_ceph-c_511015c23eef0684a64171b710e432448df2b7c7/qa/tasks/workunit.py", line 145, in task
,     _spawn_on_all_clients(ctx, refspec, all_tasks, config.get('env'),
,   File "/home/teuthworker/src/github.com_ceph_ceph-c_511015c23eef0684a64171b710e432448df2b7c7/qa/tasks/workunit.py", line 295, in _spawn_on_all_clients

Ignoring malparsed backtrace:     suppress = manager.__exit__(*exc_info)
,   File "/usr/lib/python3.8/contextlib.py", line 120, in __exit__
,     next(self.gen)

Ignoring malparsed backtrace:  ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
,  1: /lib/x86_64-linux-gnu/libpthread.so.0(+0x14420) [0x7fe43e6fe420]
,  2: pthread_mutex_lock()

Ignoring malparsed backtrace: 2023-03-24T21:56:18.804 INFO:tasks.workunit.client.0.smithi117.stdout:{
, 2023-03-24T21:56:18.805 INFO:tasks.workunit.client.0.smithi117.stdout:    "ops": [],
, 2023-03-24T21:56:18.805 INFO:tasks.workunit.client.0.smithi117.stdout:    "num_ops": 0

Ignoring malparsed backtrace: 2023-03-24T21:56:20.430 INFO:tasks.workunit.client.0.smithi117.stdout:success
, /home/ubuntu/cephtest/clone.client.0/qa/standalone/osd/divergent-priors.sh:619: TEST_divergent_2:  rm td/divergent-priors/td/divergent-priors/exp.30989.out
, rm: cannot remove 'td/divergent-priors/td/divergent-priors/exp.30989.out': No such file or directory

Ignoring malparsed backtrace:   File "/home/teuthworker/src/github.com_ceph_ceph-c_511015c23eef0684a64171b710e432448df2b7c7/qa/tasks/workunit.py", line 145, in task
,     _spawn_on_all_clients(ctx, refspec, all_tasks, config.get('env'),
,   File "/home/teuthworker/src/github.com_ceph_ceph-c_511015c23eef0684a64171b710e432448df2b7c7/qa/tasks/workunit.py", line 295, in _spawn_on_all_clients

Ignoring malparsed backtrace:     suppress = manager.__exit__(*exc_info)
,   File "/usr/lib/python3.8/contextlib.py", line 120, in __exit__
,     next(self.gen)

Ignoring malparsed backtrace:  ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
,  1: /lib/x86_64-linux-gnu/libpthread.so.0(+0x14420) [0x7fe43e6fe420]
,  2: pthread_mutex_lock()

Ignoring malparsed backtrace: 2023-03-24T21:56:18.804 INFO:tasks.workunit.client.0.smithi117.stdout:{
, 2023-03-24T21:56:18.805 INFO:tasks.workunit.client.0.smithi117.stdout:    "ops": [],
, 2023-03-24T21:56:18.805 INFO:tasks.workunit.client.0.smithi117.stdout:    "num_ops": 0

Ignoring malparsed backtrace: 2023-03-24T21:56:20.430 INFO:tasks.workunit.client.0.smithi117.stdout:success
, /home/ubuntu/cephtest/clone.client.0/qa/standalone/osd/divergent-priors.sh:619: TEST_divergent_2:  rm td/divergent-priors/td/divergent-priors/exp.30989.out
, rm: cannot remove 'td/divergent-priors/td/divergent-priors/exp.30989.out': No such file or directory

Ignoring malparsed backtrace:   File "/home/teuthworker/src/github.com_ceph_ceph-c_511015c23eef0684a64171b710e432448df2b7c7/qa/tasks/workunit.py", line 145, in task
,     _spawn_on_all_clients(ctx, refspec, all_tasks, config.get('env'),
,   File "/home/teuthworker/src/github.com_ceph_ceph-c_511015c23eef0684a64171b710e432448df2b7c7/qa/tasks/workunit.py", line 295, in _spawn_on_all_clients

Ignoring malparsed backtrace:     suppress = manager.__exit__(*exc_info)
,   File "/usr/lib/python3.8/contextlib.py", line 120, in __exit__
,     next(self.gen)

Ignoring malparsed backtrace:  ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
,  1: /lib/x86_64-linux-gnu/libpthread.so.0(+0x14420) [0x7fe43e6fe420]
,  2: pthread_mutex_lock()

Ignoring malparsed backtrace: 2023-03-24T21:56:18.804 INFO:tasks.workunit.client.0.smithi117.stdout:{
, 2023-03-24T21:56:18.805 INFO:tasks.workunit.client.0.smithi117.stdout:    "ops": [],
, 2023-03-24T21:56:18.805 INFO:tasks.workunit.client.0.smithi117.stdout:    "num_ops": 0

Ignoring malparsed backtrace: 2023-03-24T21:56:20.430 INFO:tasks.workunit.client.0.smithi117.stdout:success
, /home/ubuntu/cephtest/clone.client.0/qa/standalone/osd/divergent-priors.sh:619: TEST_divergent_2:  rm td/divergent-priors/td/divergent-priors/exp.30989.out
, rm: cannot remove 'td/divergent-priors/td/divergent-priors/exp.30989.out': No such file or directory

Ignoring malparsed backtrace:   File "/home/teuthworker/src/github.com_ceph_ceph-c_511015c23eef0684a64171b710e432448df2b7c7/qa/tasks/workunit.py", line 145, in task
,     _spawn_on_all_clients(ctx, refspec, all_tasks, config.get('env'),
,   File "/home/teuthworker/src/github.com_ceph_ceph-c_511015c23eef0684a64171b710e432448df2b7c7/qa/tasks/workunit.py", line 295, in _spawn_on_all_clients

Ignoring malparsed backtrace:     suppress = manager.__exit__(*exc_info)
,   File "/usr/lib/python3.8/contextlib.py", line 120, in __exit__
,     next(self.gen)

Ignoring malparsed backtrace:  ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
,  1: /lib/x86_64-linux-gnu/libpthread.so.0(+0x14420) [0x7f484f1d9420]
,  2: pthread_mutex_lock()

Ignoring malparsed backtrace: 2023-03-25T04:34:51.991 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump
, 2023-03-25T04:34:52.002 DEBUG:teuthology.orchestra.run.smithi139:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump
, 2023-03-25T04:34:52.049 INFO:teuthology.orchestra.run.smithi139.stdout:kernel.core_pattern = core

Ignoring malparsed backtrace:  ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
,  1: /lib/x86_64-linux-gnu/libpthread.so.0(+0x14420) [0x7f484f1d9420]
,  2: pthread_mutex_lock()

Ignoring malparsed backtrace: 2023-03-25T04:34:51.991 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump
, 2023-03-25T04:34:52.002 DEBUG:teuthology.orchestra.run.smithi139:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump
, 2023-03-25T04:34:52.049 INFO:teuthology.orchestra.run.smithi139.stdout:kernel.core_pattern = core

Ignoring malparsed backtrace:  ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
,  1: /lib/x86_64-linux-gnu/libpthread.so.0(+0x14420) [0x7f484f1d9420]
,  2: pthread_mutex_lock()

Ignoring malparsed backtrace: 2023-03-25T04:34:51.991 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump
, 2023-03-25T04:34:52.002 DEBUG:teuthology.orchestra.run.smithi139:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump
, 2023-03-25T04:34:52.049 INFO:teuthology.orchestra.run.smithi139.stdout:kernel.core_pattern = core

Ignoring malparsed backtrace:  ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
,  1: /lib/x86_64-linux-gnu/libpthread.so.0(+0x14420) [0x7f484f1d9420]
,  2: pthread_mutex_lock()

Ignoring malparsed backtrace: 2023-03-25T04:34:51.991 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump
, 2023-03-25T04:34:52.002 DEBUG:teuthology.orchestra.run.smithi139:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump
, 2023-03-25T04:34:52.049 INFO:teuthology.orchestra.run.smithi139.stdout:kernel.core_pattern = core

Ignoring malparsed backtrace:  ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
,  1: /lib/x86_64-linux-gnu/libpthread.so.0(+0x14420) [0x7f484f1d9420]
,  2: pthread_mutex_lock()

Ignoring malparsed backtrace: 2023-03-25T04:34:51.991 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump
, 2023-03-25T04:34:52.002 DEBUG:teuthology.orchestra.run.smithi139:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump
, 2023-03-25T04:34:52.049 INFO:teuthology.orchestra.run.smithi139.stdout:kernel.core_pattern = core

Ignoring malparsed backtrace:  ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
,  1: /lib/x86_64-linux-gnu/libpthread.so.0(+0x14420) [0x7f484f1d9420]
,  2: pthread_mutex_lock()

Ignoring malparsed backtrace: 2023-03-25T04:34:51.991 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump
, 2023-03-25T04:34:52.002 DEBUG:teuthology.orchestra.run.smithi139:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump
, 2023-03-25T04:34:52.049 INFO:teuthology.orchestra.run.smithi139.stdout:kernel.core_pattern = core

Ignoring malparsed backtrace:  ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
,  1: /lib/x86_64-linux-gnu/libpthread.so.0(+0x14420) [0x7f484f1d9420]
,  2: pthread_mutex_lock()

Ignoring malparsed backtrace: 2023-03-25T04:34:51.991 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump
, 2023-03-25T04:34:52.002 DEBUG:teuthology.orchestra.run.smithi139:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump
, 2023-03-25T04:34:52.049 INFO:teuthology.orchestra.run.smithi139.stdout:kernel.core_pattern = core

Ignoring malparsed backtrace:  ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
,  1: /lib/x86_64-linux-gnu/libpthread.so.0(+0x14420) [0x7f484f1d9420]
,  2: pthread_mutex_lock()

Ignoring malparsed backtrace: 2023-03-25T04:34:51.991 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump
, 2023-03-25T04:34:52.002 DEBUG:teuthology.orchestra.run.smithi139:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump
, 2023-03-25T04:34:52.049 INFO:teuthology.orchestra.run.smithi139.stdout:kernel.core_pattern = core

Ignoring malparsed backtrace:  ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
,  1: /lib/x86_64-linux-gnu/libpthread.so.0(+0x14420) [0x7f484f1d9420]
,  2: pthread_mutex_lock()

Ignoring malparsed backtrace: 2023-03-25T04:34:51.991 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump
, 2023-03-25T04:34:52.002 DEBUG:teuthology.orchestra.run.smithi139:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump
, 2023-03-25T04:34:52.049 INFO:teuthology.orchestra.run.smithi139.stdout:kernel.core_pattern = core

Ignoring malparsed backtrace:  ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
,  1: /lib/x86_64-linux-gnu/libpthread.so.0(+0x14420) [0x7f484f1d9420]
,  2: pthread_mutex_lock()

Ignoring malparsed backtrace: 2023-03-25T04:34:51.991 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump
, 2023-03-25T04:34:52.002 DEBUG:teuthology.orchestra.run.smithi139:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump
, 2023-03-25T04:34:52.049 INFO:teuthology.orchestra.run.smithi139.stdout:kernel.core_pattern = core

Ignoring malparsed backtrace:  ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
,  1: /lib/x86_64-linux-gnu/libpthread.so.0(+0x14420) [0x7f484f1d9420]
,  2: pthread_mutex_lock()

Ignoring malparsed backtrace: 2023-03-25T04:34:51.991 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump
, 2023-03-25T04:34:52.002 DEBUG:teuthology.orchestra.run.smithi139:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump
, 2023-03-25T04:34:52.049 INFO:teuthology.orchestra.run.smithi139.stdout:kernel.core_pattern = core

Ignoring malparsed backtrace:  ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
,  1: /lib/x86_64-linux-gnu/libpthread.so.0(+0x14420) [0x7f484f1d9420]
,  2: pthread_mutex_lock()

Ignoring malparsed backtrace: 2023-03-25T04:34:51.991 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump
, 2023-03-25T04:34:52.002 DEBUG:teuthology.orchestra.run.smithi139:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump
, 2023-03-25T04:34:52.049 INFO:teuthology.orchestra.run.smithi139.stdout:kernel.core_pattern = core

Ignoring malparsed backtrace:  ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
,  1: /lib/x86_64-linux-gnu/libpthread.so.0(+0x14420) [0x7f484f1d9420]
,  2: pthread_mutex_lock()

Ignoring malparsed backtrace: 2023-03-25T04:34:51.991 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump
, 2023-03-25T04:34:52.002 DEBUG:teuthology.orchestra.run.smithi139:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump
, 2023-03-25T04:34:52.049 INFO:teuthology.orchestra.run.smithi139.stdout:kernel.core_pattern = core

Ignoring malparsed backtrace:  ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
,  1: /lib/x86_64-linux-gnu/libpthread.so.0(+0x14420) [0x7f484f1d9420]
,  2: pthread_mutex_lock()

Ignoring malparsed backtrace: 2023-03-25T04:34:51.991 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump
, 2023-03-25T04:34:52.002 DEBUG:teuthology.orchestra.run.smithi139:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump
, 2023-03-25T04:34:52.049 INFO:teuthology.orchestra.run.smithi139.stdout:kernel.core_pattern = core

Ignoring malparsed backtrace:  ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
,  1: /lib/x86_64-linux-gnu/libpthread.so.0(+0x14420) [0x7f484f1d9420]
,  2: pthread_mutex_lock()

Ignoring malparsed backtrace: 2023-03-25T04:34:51.991 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump
, 2023-03-25T04:34:52.002 DEBUG:teuthology.orchestra.run.smithi139:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump
, 2023-03-25T04:34:52.049 INFO:teuthology.orchestra.run.smithi139.stdout:kernel.core_pattern = core

Ignoring malparsed backtrace:  ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
,  1: /lib/x86_64-linux-gnu/libpthread.so.0(+0x14420) [0x7f484f1d9420]
,  2: pthread_mutex_lock()

Ignoring malparsed backtrace: 2023-03-25T04:34:51.991 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump
, 2023-03-25T04:34:52.002 DEBUG:teuthology.orchestra.run.smithi139:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump
, 2023-03-25T04:34:52.049 INFO:teuthology.orchestra.run.smithi139.stdout:kernel.core_pattern = core

Ignoring malparsed backtrace:  ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
,  1: /lib/x86_64-linux-gnu/libpthread.so.0(+0x14420) [0x7f484f1d9420]
,  2: pthread_mutex_lock()

Ignoring malparsed backtrace: 2023-03-25T04:34:51.991 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump
, 2023-03-25T04:34:52.002 DEBUG:teuthology.orchestra.run.smithi139:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump
, 2023-03-25T04:34:52.049 INFO:teuthology.orchestra.run.smithi139.stdout:kernel.core_pattern = core

Ignoring malparsed backtrace:  ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
,  1: /lib/x86_64-linux-gnu/libpthread.so.0(+0x14420) [0x7f484f1d9420]
,  2: pthread_mutex_lock()

Ignoring malparsed backtrace: 2023-03-25T04:34:51.991 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump
, 2023-03-25T04:34:52.002 DEBUG:teuthology.orchestra.run.smithi139:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump
, 2023-03-25T04:34:52.049 INFO:teuthology.orchestra.run.smithi139.stdout:kernel.core_pattern = core

Ignoring malparsed backtrace:  ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
,  1: /lib/x86_64-linux-gnu/libpthread.so.0(+0x14420) [0x7f484f1d9420]
,  2: pthread_mutex_lock()

Ignoring malparsed backtrace: 2023-03-25T04:34:51.991 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump
, 2023-03-25T04:34:52.002 DEBUG:teuthology.orchestra.run.smithi139:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump
, 2023-03-25T04:34:52.049 INFO:teuthology.orchestra.run.smithi139.stdout:kernel.core_pattern = core

Ignoring malparsed backtrace:  ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
,  1: /lib/x86_64-linux-gnu/libpthread.so.0(+0x14420) [0x7f484f1d9420]
,  2: pthread_mutex_lock()

Ignoring malparsed backtrace: 2023-03-25T04:34:51.991 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump
, 2023-03-25T04:34:52.002 DEBUG:teuthology.orchestra.run.smithi139:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump
, 2023-03-25T04:34:52.049 INFO:teuthology.orchestra.run.smithi139.stdout:kernel.core_pattern = core

Ignoring malparsed backtrace:  ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
,  1: /lib64/libpthread.so.0(+0x12ce0) [0x7fce85a02ce0]
,  2: __pthread_mutex_lock()

Ignoring malparsed backtrace: 2023-03-25T03:04:01.973 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd deep-scrub 1
, instructed osd(s) 1 to deep-scrub
, 2023-03-25T03:04:02.304 INFO:tasks.ceph:Scrubbing osd.2

Ignoring malparsed backtrace: 2023-03-25T03:04:25.957 INFO:teuthology.task.internal.syslog:Checking logs for errors...
, 2023-03-25T03:04:25.957 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi090.front.sepia.ceph.com
, 2023-03-25T03:04:25.958 DEBUG:teuthology.orchestra.run.smithi090:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1

Ignoring malparsed backtrace:  ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
,  1: /lib64/libpthread.so.0(+0x12ce0) [0x7fce85a02ce0]
,  2: __pthread_mutex_lock()

Ignoring malparsed backtrace: 2023-03-25T03:04:01.973 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd deep-scrub 1
, instructed osd(s) 1 to deep-scrub
, 2023-03-25T03:04:02.304 INFO:tasks.ceph:Scrubbing osd.2

Ignoring malparsed backtrace: 2023-03-25T03:04:25.957 INFO:teuthology.task.internal.syslog:Checking logs for errors...
, 2023-03-25T03:04:25.957 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi090.front.sepia.ceph.com
, 2023-03-25T03:04:25.958 DEBUG:teuthology.orchestra.run.smithi090:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1

Ignoring malparsed backtrace:  ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
,  1: /lib64/libpthread.so.0(+0x12ce0) [0x7fce85a02ce0]
,  2: __pthread_mutex_lock()

Ignoring malparsed backtrace: 2023-03-25T03:04:01.973 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd deep-scrub 1
, instructed osd(s) 1 to deep-scrub
, 2023-03-25T03:04:02.304 INFO:tasks.ceph:Scrubbing osd.2

Ignoring malparsed backtrace: 2023-03-25T03:04:25.957 INFO:teuthology.task.internal.syslog:Checking logs for errors...
, 2023-03-25T03:04:25.957 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi090.front.sepia.ceph.com
, 2023-03-25T03:04:25.958 DEBUG:teuthology.orchestra.run.smithi090:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1

Ignoring malparsed backtrace:  ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
,  1: /lib64/libpthread.so.0(+0x12ce0) [0x7fce85a02ce0]
,  2: __pthread_mutex_lock()

Ignoring malparsed backtrace: 2023-03-25T03:04:01.973 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd deep-scrub 1
, instructed osd(s) 1 to deep-scrub
, 2023-03-25T03:04:02.304 INFO:tasks.ceph:Scrubbing osd.2

Ignoring malparsed backtrace: 2023-03-25T03:04:25.957 INFO:teuthology.task.internal.syslog:Checking logs for errors...
, 2023-03-25T03:04:25.957 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi090.front.sepia.ceph.com
, 2023-03-25T03:04:25.958 DEBUG:teuthology.orchestra.run.smithi090:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1

Ignoring malparsed backtrace:  ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
,  1: /lib64/libpthread.so.0(+0x12ce0) [0x7fcfed5edce0]
,  2: __pthread_mutex_lock()

Ignoring malparsed backtrace: 2023-03-25T04:57:02.922 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd deep-scrub 1
, instructed osd(s) 1 to deep-scrub
, 2023-03-25T04:57:03.254 INFO:tasks.ceph:Scrubbing osd.2

Ignoring malparsed backtrace: 2023-03-25T04:57:39.511 DEBUG:teuthology.orchestra.run.smithi083:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true
, bash: ntpq: command not found
, 2023-03-25T04:57:39.532 INFO:teuthology.orchestra.run.smithi083.stdout:MS Name/IP address         Stratum Poll Reach LastRx Last sample

Ignoring malparsed backtrace:  ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
,  1: /lib64/libpthread.so.0(+0x12ce0) [0x7fcfed5edce0]
,  2: __pthread_mutex_lock()

Ignoring malparsed backtrace: 2023-03-25T04:57:02.922 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd deep-scrub 1
, instructed osd(s) 1 to deep-scrub
, 2023-03-25T04:57:03.254 INFO:tasks.ceph:Scrubbing osd.2

Ignoring malparsed backtrace: 2023-03-25T04:57:39.511 DEBUG:teuthology.orchestra.run.smithi083:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true
, bash: ntpq: command not found
, 2023-03-25T04:57:39.532 INFO:teuthology.orchestra.run.smithi083.stdout:MS Name/IP address         Stratum Poll Reach LastRx Last sample

Ignoring malparsed backtrace:  ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
,  1: /lib64/libpthread.so.0(+0x12ce0) [0x7fcfed5edce0]
,  2: __pthread_mutex_lock()

Ignoring malparsed backtrace: 2023-03-25T04:57:02.922 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd deep-scrub 1
, instructed osd(s) 1 to deep-scrub
, 2023-03-25T04:57:03.254 INFO:tasks.ceph:Scrubbing osd.2

Ignoring malparsed backtrace: 2023-03-25T04:57:39.511 DEBUG:teuthology.orchestra.run.smithi083:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true
, bash: ntpq: command not found
, 2023-03-25T04:57:39.532 INFO:teuthology.orchestra.run.smithi083.stdout:MS Name/IP address         Stratum Poll Reach LastRx Last sample

Ignoring malparsed backtrace:  ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
,  1: /lib64/libpthread.so.0(+0x12ce0) [0x7fcfed5edce0]
,  2: __pthread_mutex_lock()

Ignoring malparsed backtrace: 2023-03-25T04:57:02.922 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd deep-scrub 1
, instructed osd(s) 1 to deep-scrub
, 2023-03-25T04:57:03.254 INFO:tasks.ceph:Scrubbing osd.2

Ignoring malparsed backtrace: 2023-03-25T04:57:39.511 DEBUG:teuthology.orchestra.run.smithi083:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true
, bash: ntpq: command not found
, 2023-03-25T04:57:39.532 INFO:teuthology.orchestra.run.smithi083.stdout:MS Name/IP address         Stratum Poll Reach LastRx Last sample

Ignoring malparsed backtrace:  ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
,  1: /lib64/libpthread.so.0(+0x12ce0) [0x7f3524b93ce0]
,  2: __pthread_mutex_lock()

Ignoring malparsed backtrace: 2023-03-25T12:21:20.604 DEBUG:teuthology.orchestra.run.smithi097:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd deep-scrub 1
, instructed osd(s) 1 to deep-scrub
, 2023-03-25T12:21:20.939 INFO:tasks.ceph:Scrubbing osd.2

Ignoring malparsed backtrace: 2023-03-25T12:21:44.600 INFO:teuthology.task.internal.syslog:Checking logs for errors...
, 2023-03-25T12:21:44.601 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi097.front.sepia.ceph.com
, 2023-03-25T12:21:44.601 DEBUG:teuthology.orchestra.run.smithi097:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1

Ignoring malparsed backtrace:  ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
,  1: /lib64/libpthread.so.0(+0x12ce0) [0x7f3524b93ce0]
,  2: __pthread_mutex_lock()

Ignoring malparsed backtrace: 2023-03-25T12:21:20.604 DEBUG:teuthology.orchestra.run.smithi097:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd deep-scrub 1
, instructed osd(s) 1 to deep-scrub
, 2023-03-25T12:21:20.939 INFO:tasks.ceph:Scrubbing osd.2

Ignoring malparsed backtrace: 2023-03-25T12:21:44.600 INFO:teuthology.task.internal.syslog:Checking logs for errors...
, 2023-03-25T12:21:44.601 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi097.front.sepia.ceph.com
, 2023-03-25T12:21:44.601 DEBUG:teuthology.orchestra.run.smithi097:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1

Ignoring malparsed backtrace:  ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
,  1: /lib64/libpthread.so.0(+0x12ce0) [0x7f3524b93ce0]
,  2: __pthread_mutex_lock()

Ignoring malparsed backtrace: 2023-03-25T12:21:20.604 DEBUG:teuthology.orchestra.run.smithi097:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd deep-scrub 1
, instructed osd(s) 1 to deep-scrub
, 2023-03-25T12:21:20.939 INFO:tasks.ceph:Scrubbing osd.2

Ignoring malparsed backtrace: 2023-03-25T12:21:44.600 INFO:teuthology.task.internal.syslog:Checking logs for errors...
, 2023-03-25T12:21:44.601 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi097.front.sepia.ceph.com
, 2023-03-25T12:21:44.601 DEBUG:teuthology.orchestra.run.smithi097:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1

Ignoring malparsed backtrace:  ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
,  1: /lib64/libpthread.so.0(+0x12ce0) [0x7f3524b93ce0]
,  2: __pthread_mutex_lock()

Ignoring malparsed backtrace: 2023-03-25T12:21:20.604 DEBUG:teuthology.orchestra.run.smithi097:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd deep-scrub 1
, instructed osd(s) 1 to deep-scrub
, 2023-03-25T12:21:20.939 INFO:tasks.ceph:Scrubbing osd.2

Ignoring malparsed backtrace: 2023-03-25T12:21:44.600 INFO:teuthology.task.internal.syslog:Checking logs for errors...
, 2023-03-25T12:21:44.601 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi097.front.sepia.ceph.com
, 2023-03-25T12:21:44.601 DEBUG:teuthology.orchestra.run.smithi097:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1

Ignoring malparsed backtrace:  ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
,  1: /lib64/libpthread.so.0(+0x12cf0) [0x7f1453c2fcf0]
,  2: __pthread_mutex_lock()

Ignoring malparsed backtrace: 2023-03-25T04:28:05.172 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":142,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":5,"modified":"2023-03-25T04:27:12.092098+0000","services":{}},"progress_events":{}}
, 2023-03-25T04:28:05.185 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now
, 2023-03-25T04:28:05.186 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace: 2023-03-25T04:28:28.360 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":116,"stamp":"2023-03-25T04:28:28.108007+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002578"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.314661+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:27:45.314661+0000","last_peered":"2023-03-25T04:27:45.314661+0000","last_clean":"2023-03-25T04:27:45.314661+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:27:45.314661+0000","last_undegraded":"2023-03-25T04:27:45.314661+0000","last_fullsized":"2023-03-25T04:27:45.314661+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":25,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.310745+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:27:44.310745+0000","last_peered":"2023-03-25T04:27:44.310745+0000","last_clean":"2023-03-25T04:27:44.310745+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:27:44.310745+0000","last_undegraded":"2023-03-25T04:27:44.310745+0000","last_fullsized":"2023-03-25T04:27:44.310745+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":25,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.310650+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:27:44.310650+0000","last_peered":"2023-03-25T04:27:44.310650+0000","last_clean":"2023-03-25T04:27:44.310650+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:27:44.310650+0000","last_undegraded":"2023-03-25T04:27:44.310650+0000","last_fullsized":"2023-03-25T04:27:44.310650+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":26,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.312962+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:27:44.312962+0000","last_peered":"2023-03-25T04:27:44.312962+0000","last_clean":"2023-03-25T04:27:44.312962+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:27:44.312962+0000","last_undegraded":"2023-03-25T04:27:44.312962+0000","last_fullsized":"2023-03-25T04:27:44.312962+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315064+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:27:45.315064+0000","last_peered":"2023-03-25T04:27:45.315064+0000","last_clean":"2023-03-25T04:27:45.315064+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:27:45.315064+0000","last_undegraded":"2023-03-25T04:27:45.315064+0000","last_fullsized":"2023-03-25T04:27:45.315064+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":25,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.310707+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:27:44.310707+0000","last_peered":"2023-03-25T04:27:44.310707+0000","last_clean":"2023-03-25T04:27:44.310707+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:27:44.310707+0000","last_undegraded":"2023-03-25T04:27:44.310707+0000","last_fullsized":"2023-03-25T04:27:44.310707+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117007,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.375}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051597,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.186}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:28:28.362 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean
, 2023-03-25T04:28:28.362 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:28:45.655 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":124,"stamp":"2023-03-25T04:28:44.111265+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002492"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.314661+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:27:45.314661+0000","last_peered":"2023-03-25T04:27:45.314661+0000","last_clean":"2023-03-25T04:27:45.314661+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:27:45.314661+0000","last_undegraded":"2023-03-25T04:27:45.314661+0000","last_fullsized":"2023-03-25T04:27:45.314661+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":25,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.310745+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:27:44.310745+0000","last_peered":"2023-03-25T04:27:44.310745+0000","last_clean":"2023-03-25T04:27:44.310745+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:27:44.310745+0000","last_undegraded":"2023-03-25T04:27:44.310745+0000","last_fullsized":"2023-03-25T04:27:44.310745+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":25,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.310650+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:27:44.310650+0000","last_peered":"2023-03-25T04:27:44.310650+0000","last_clean":"2023-03-25T04:27:44.310650+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:27:44.310650+0000","last_undegraded":"2023-03-25T04:27:44.310650+0000","last_fullsized":"2023-03-25T04:27:44.310650+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":26,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.312962+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:27:44.312962+0000","last_peered":"2023-03-25T04:27:44.312962+0000","last_clean":"2023-03-25T04:27:44.312962+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:27:44.312962+0000","last_undegraded":"2023-03-25T04:27:44.312962+0000","last_fullsized":"2023-03-25T04:27:44.312962+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315064+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:27:45.315064+0000","last_peered":"2023-03-25T04:27:45.315064+0000","last_clean":"2023-03-25T04:27:45.315064+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:27:45.315064+0000","last_undegraded":"2023-03-25T04:27:45.315064+0000","last_fullsized":"2023-03-25T04:27:45.315064+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":25,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.310707+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:27:44.310707+0000","last_peered":"2023-03-25T04:27:44.310707+0000","last_clean":"2023-03-25T04:27:44.310707+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:27:44.310707+0000","last_undegraded":"2023-03-25T04:27:44.310707+0000","last_fullsized":"2023-03-25T04:27:44.310707+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117011,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:28:31 2023","interfaces":[{"interface":"back","average":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.36899999999999999},{"interface":"front","average":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"min":{"1min":0.19,"5min":0.19,"15min":0.19},"max":{"1min":0.46300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.39800000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051601,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:28:24 2023","interfaces":[{"interface":"back","average":{"1min":0.249,"5min":0.249,"15min":0.249},"min":{"1min":0.17699999999999999,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"last":0.254},{"interface":"front","average":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"min":{"1min":0.20200000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"last":0.312}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:28:45.656 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace: 2023-03-25T04:29:05.171 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}
, 2023-03-25T04:29:05.171 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json
, 2023-03-25T04:29:05.561 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:29:22.465 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:29:22.773 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all

Ignoring malparsed backtrace: 2023-03-25T04:29:42.405 INFO:teuthology.orchestra.run.smithi070.stdout:
, 2023-03-25T04:29:42.406 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":240,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":9,"modified":"2023-03-25T04:29:32.120358+0000","services":{"mon":{"daemons":{"summary":"","a":{"start_epoch":0,"start_stamp":"0.000000","gid":0,"addr":"(unrecognized address family 0)/0","metadata":{},"task_status":{}}}}}},"progress_events":{}}
, 2023-03-25T04:29:42.419 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:29:59.627 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":161,"stamp":"2023-03-25T04:29:58.125186+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002398"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315468+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:27:45.315468+0000","last_peered":"2023-03-25T04:27:45.315468+0000","last_clean":"2023-03-25T04:27:45.315468+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:27:45.315468+0000","last_undegraded":"2023-03-25T04:27:45.315468+0000","last_fullsized":"2023-03-25T04:27:45.315468+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312845+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:27:45.312845+0000","last_peered":"2023-03-25T04:27:45.312845+0000","last_clean":"2023-03-25T04:27:45.312845+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:27:45.312845+0000","last_undegraded":"2023-03-25T04:27:45.312845+0000","last_fullsized":"2023-03-25T04:27:45.312845+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312882+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:27:45.312882+0000","last_peered":"2023-03-25T04:27:45.312882+0000","last_clean":"2023-03-25T04:27:45.312882+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:27:45.312882+0000","last_undegraded":"2023-03-25T04:27:45.312882+0000","last_fullsized":"2023-03-25T04:27:45.312882+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.311730+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:27:45.311730+0000","last_peered":"2023-03-25T04:27:45.311730+0000","last_clean":"2023-03-25T04:27:45.311730+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:27:45.311730+0000","last_undegraded":"2023-03-25T04:27:45.311730+0000","last_fullsized":"2023-03-25T04:27:45.311730+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315735+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:27:45.315735+0000","last_peered":"2023-03-25T04:27:45.315735+0000","last_clean":"2023-03-25T04:27:45.315735+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:27:45.315735+0000","last_undegraded":"2023-03-25T04:27:45.315735+0000","last_fullsized":"2023-03-25T04:27:45.315735+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312833+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:27:45.312833+0000","last_peered":"2023-03-25T04:27:45.312833+0000","last_clean":"2023-03-25T04:27:45.312833+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:27:45.312833+0000","last_undegraded":"2023-03-25T04:27:45.312833+0000","last_fullsized":"2023-03-25T04:27:45.312833+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117026,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:29:33 2023","interfaces":[{"interface":"back","average":{"1min":0.26300000000000001,"5min":0.29099999999999998,"15min":0.29599999999999999},"min":{"1min":0.191,"5min":0.191,"15min":0.191},"max":{"1min":0.371,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.20499999999999999},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.29599999999999999,"15min":0.29999999999999999},"min":{"1min":0.21099999999999999,"5min":0.19,"15min":0.19},"max":{"1min":0.39800000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.19500000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051616,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:29:29 2023","interfaces":[{"interface":"back","average":{"1min":0.27200000000000002,"5min":0.253,"15min":0.25},"min":{"1min":0.183,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"last":0.26400000000000001},{"interface":"front","average":{"1min":0.28499999999999998,"5min":0.26800000000000002,"15min":0.26500000000000001},"min":{"1min":0.16600000000000001,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"last":0.29899999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:29:59.629 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean

Ignoring malparsed backtrace: 2023-03-25T04:30:16.889 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:30:16.903 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":170,"stamp":"2023-03-25T04:30:16.128427+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002222"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315468+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:27:45.315468+0000","last_peered":"2023-03-25T04:27:45.315468+0000","last_clean":"2023-03-25T04:27:45.315468+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:27:45.315468+0000","last_undegraded":"2023-03-25T04:27:45.315468+0000","last_fullsized":"2023-03-25T04:27:45.315468+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312845+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:27:45.312845+0000","last_peered":"2023-03-25T04:27:45.312845+0000","last_clean":"2023-03-25T04:27:45.312845+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:27:45.312845+0000","last_undegraded":"2023-03-25T04:27:45.312845+0000","last_fullsized":"2023-03-25T04:27:45.312845+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312882+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:27:45.312882+0000","last_peered":"2023-03-25T04:27:45.312882+0000","last_clean":"2023-03-25T04:27:45.312882+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:27:45.312882+0000","last_undegraded":"2023-03-25T04:27:45.312882+0000","last_fullsized":"2023-03-25T04:27:45.312882+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.311730+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:27:45.311730+0000","last_peered":"2023-03-25T04:27:45.311730+0000","last_clean":"2023-03-25T04:27:45.311730+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:27:45.311730+0000","last_undegraded":"2023-03-25T04:27:45.311730+0000","last_fullsized":"2023-03-25T04:27:45.311730+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315735+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:27:45.315735+0000","last_peered":"2023-03-25T04:27:45.315735+0000","last_clean":"2023-03-25T04:27:45.315735+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:27:45.315735+0000","last_undegraded":"2023-03-25T04:27:45.315735+0000","last_fullsized":"2023-03-25T04:27:45.315735+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312833+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:27:45.312833+0000","last_peered":"2023-03-25T04:27:45.312833+0000","last_clean":"2023-03-25T04:27:45.312833+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:27:45.312833+0000","last_undegraded":"2023-03-25T04:27:45.312833+0000","last_fullsized":"2023-03-25T04:27:45.312833+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117030,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:29:33 2023","interfaces":[{"interface":"back","average":{"1min":0.26300000000000001,"5min":0.29099999999999998,"15min":0.29599999999999999},"min":{"1min":0.191,"5min":0.191,"15min":0.191},"max":{"1min":0.371,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.245},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.29599999999999999,"15min":0.29999999999999999},"min":{"1min":0.21099999999999999,"5min":0.19,"15min":0.19},"max":{"1min":0.39800000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.20899999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051620,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:29:29 2023","interfaces":[{"interface":"back","average":{"1min":0.27200000000000002,"5min":0.253,"15min":0.25},"min":{"1min":0.183,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"last":0.19400000000000001},{"interface":"front","average":{"1min":0.28499999999999998,"5min":0.26800000000000002,"15min":0.26500000000000001},"min":{"1min":0.16600000000000001,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"last":0.23599999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:30:31.984+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:30:32.201+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:30:32.981+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)

Ignoring malparsed backtrace: 2023-03-25T04:30:48.449 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now
, 2023-03-25T04:30:48.450 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:30:48.760 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:31:08.342 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":196,"stamp":"2023-03-25T04:31:08.139018+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002431"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315468+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:27:45.315468+0000","last_peered":"2023-03-25T04:27:45.315468+0000","last_clean":"2023-03-25T04:27:45.315468+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:27:45.315468+0000","last_undegraded":"2023-03-25T04:27:45.315468+0000","last_fullsized":"2023-03-25T04:27:45.315468+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312845+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:27:45.312845+0000","last_peered":"2023-03-25T04:27:45.312845+0000","last_clean":"2023-03-25T04:27:45.312845+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:27:45.312845+0000","last_undegraded":"2023-03-25T04:27:45.312845+0000","last_fullsized":"2023-03-25T04:27:45.312845+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312882+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:27:45.312882+0000","last_peered":"2023-03-25T04:27:45.312882+0000","last_clean":"2023-03-25T04:27:45.312882+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:27:45.312882+0000","last_undegraded":"2023-03-25T04:27:45.312882+0000","last_fullsized":"2023-03-25T04:27:45.312882+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.311730+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:27:45.311730+0000","last_peered":"2023-03-25T04:27:45.311730+0000","last_clean":"2023-03-25T04:27:45.311730+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:27:45.311730+0000","last_undegraded":"2023-03-25T04:27:45.311730+0000","last_fullsized":"2023-03-25T04:27:45.311730+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315735+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:27:45.315735+0000","last_peered":"2023-03-25T04:27:45.315735+0000","last_clean":"2023-03-25T04:27:45.315735+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:27:45.315735+0000","last_undegraded":"2023-03-25T04:27:45.315735+0000","last_fullsized":"2023-03-25T04:27:45.315735+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312833+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:27:45.312833+0000","last_peered":"2023-03-25T04:27:45.312833+0000","last_clean":"2023-03-25T04:27:45.312833+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:27:45.312833+0000","last_undegraded":"2023-03-25T04:27:45.312833+0000","last_fullsized":"2023-03-25T04:27:45.312833+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117040,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:30:37 2023","interfaces":[{"interface":"back","average":{"1min":0.28799999999999998,"5min":0.28899999999999998,"15min":0.29499999999999998},"min":{"1min":0.20200000000000001,"5min":0.191,"15min":0.191},"max":{"1min":0.433,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.29799999999999999},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.28999999999999998,"15min":0.29799999999999999},"min":{"1min":0.17499999999999999,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":0.40300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.25700000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051630,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:30:32 2023","interfaces":[{"interface":"back","average":{"1min":0.25600000000000001,"5min":0.255,"15min":0.251},"min":{"1min":0.19400000000000001,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.35699999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"last":0.22800000000000001},{"interface":"front","average":{"1min":0.28699999999999998,"5min":0.27200000000000002,"15min":0.26600000000000001},"min":{"1min":0.20599999999999999,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.39400000000000002,"5min":0.42899999999999999,"15min":0.42899999999999999},"last":0.24099999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:31:08.343 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace: 2023-03-25T04:31:24.283+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:31:25.192+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:31:25.295+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)

Ignoring malparsed backtrace: 2023-03-25T04:31:40.257 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:31:40.271 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":212,"stamp":"2023-03-25T04:31:40.144732+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002208"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315468+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:27:45.315468+0000","last_peered":"2023-03-25T04:27:45.315468+0000","last_clean":"2023-03-25T04:27:45.315468+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:27:45.315468+0000","last_undegraded":"2023-03-25T04:27:45.315468+0000","last_fullsized":"2023-03-25T04:27:45.315468+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312845+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:27:45.312845+0000","last_peered":"2023-03-25T04:27:45.312845+0000","last_clean":"2023-03-25T04:27:45.312845+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:27:45.312845+0000","last_undegraded":"2023-03-25T04:27:45.312845+0000","last_fullsized":"2023-03-25T04:27:45.312845+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:23.308607+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:29:23.308607+0000","last_peered":"2023-03-25T04:29:23.308607+0000","last_clean":"2023-03-25T04:29:23.308607+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:29:23.308607+0000","last_undegraded":"2023-03-25T04:29:23.308607+0000","last_fullsized":"2023-03-25T04:29:23.308607+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":28,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:20.554710+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:29:20.554710+0000","last_peered":"2023-03-25T04:29:20.554710+0000","last_clean":"2023-03-25T04:29:20.554710+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:29:20.554710+0000","last_undegraded":"2023-03-25T04:29:20.554710+0000","last_fullsized":"2023-03-25T04:29:20.554710+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309216+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:29:28.309216+0000","last_peered":"2023-03-25T04:29:28.309216+0000","last_clean":"2023-03-25T04:29:28.309216+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:29:28.309216+0000","last_undegraded":"2023-03-25T04:29:28.309216+0000","last_fullsized":"2023-03-25T04:29:28.309216+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309233+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:29:28.309233+0000","last_peered":"2023-03-25T04:29:28.309233+0000","last_clean":"2023-03-25T04:29:28.309233+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:29:28.309233+0000","last_undegraded":"2023-03-25T04:29:28.309233+0000","last_fullsized":"2023-03-25T04:29:28.309233+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117046,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:30:37 2023","interfaces":[{"interface":"back","average":{"1min":0.28799999999999998,"5min":0.28899999999999998,"15min":0.29499999999999998},"min":{"1min":0.20200000000000001,"5min":0.191,"15min":0.191},"max":{"1min":0.433,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.216},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.28999999999999998,"15min":0.29799999999999999},"min":{"1min":0.17499999999999999,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":0.40300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.24299999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051637,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:31:35 2023","interfaces":[{"interface":"back","average":{"1min":0.26100000000000001,"5min":0.25700000000000001,"15min":0.251},"min":{"1min":0.19,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.35899999999999999,"5min":0.41599999999999998,"15min":0.41599999999999998},"last":0.221},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.27400000000000002,"15min":0.26700000000000002},"min":{"1min":0.20200000000000001,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.374,"5min":0.42899999999999999,"15min":0.42899999999999999},"last":0.24099999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:32:01.906+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:32:02.388+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:32:02.794 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:32:22.326 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":233,"stamp":"2023-03-25T04:32:22.153289+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002545"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:33.309748+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:29:33.309748+0000","last_peered":"2023-03-25T04:29:33.309748+0000","last_clean":"2023-03-25T04:29:33.309748+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:29:33.309748+0000","last_undegraded":"2023-03-25T04:29:33.309748+0000","last_fullsized":"2023-03-25T04:29:33.309748+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:33.309729+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:29:33.309729+0000","last_peered":"2023-03-25T04:29:33.309729+0000","last_clean":"2023-03-25T04:29:33.309729+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:29:33.309729+0000","last_undegraded":"2023-03-25T04:29:33.309729+0000","last_fullsized":"2023-03-25T04:29:33.309729+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:23.308607+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:29:23.308607+0000","last_peered":"2023-03-25T04:29:23.308607+0000","last_clean":"2023-03-25T04:29:23.308607+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:29:23.308607+0000","last_undegraded":"2023-03-25T04:29:23.308607+0000","last_fullsized":"2023-03-25T04:29:23.308607+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":28,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:20.554710+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:29:20.554710+0000","last_peered":"2023-03-25T04:29:20.554710+0000","last_clean":"2023-03-25T04:29:20.554710+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:29:20.554710+0000","last_undegraded":"2023-03-25T04:29:20.554710+0000","last_fullsized":"2023-03-25T04:29:20.554710+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309216+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:29:28.309216+0000","last_peered":"2023-03-25T04:29:28.309216+0000","last_clean":"2023-03-25T04:29:28.309216+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:29:28.309216+0000","last_undegraded":"2023-03-25T04:29:28.309216+0000","last_fullsized":"2023-03-25T04:29:28.309216+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309233+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:29:28.309233+0000","last_peered":"2023-03-25T04:29:28.309233+0000","last_clean":"2023-03-25T04:29:28.309233+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:29:28.309233+0000","last_undegraded":"2023-03-25T04:29:28.309233+0000","last_fullsized":"2023-03-25T04:29:28.309233+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117055,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:31:42 2023","interfaces":[{"interface":"back","average":{"1min":0.27500000000000002,"5min":0.28399999999999997,"15min":0.29399999999999998},"min":{"1min":0.19800000000000001,"5min":0.191,"15min":0.191},"max":{"1min":0.36899999999999999,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.32300000000000001},{"interface":"front","average":{"1min":0.27900000000000003,"5min":0.28499999999999998,"15min":0.29599999999999999},"min":{"1min":0.216,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":0.40799999999999997,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.27100000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051646,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:31:35 2023","interfaces":[{"interface":"back","average":{"1min":0.26100000000000001,"5min":0.25700000000000001,"15min":0.251},"min":{"1min":0.19,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.35899999999999999,"5min":0.41599999999999998,"15min":0.41599999999999998},"last":0.25600000000000001},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.27400000000000002,"15min":0.26700000000000002},"min":{"1min":0.20200000000000001,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.374,"5min":0.42899999999999999,"15min":0.42899999999999999},"last":0.29199999999999998}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:32:22.327 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean
, 2023-03-25T04:32:22.327 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}

Ignoring malparsed backtrace: 2023-03-25T04:32:39.836 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:32:39.849 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":241,"stamp":"2023-03-25T04:32:38.156312+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002346"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:33.309748+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:29:33.309748+0000","last_peered":"2023-03-25T04:29:33.309748+0000","last_clean":"2023-03-25T04:29:33.309748+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:29:33.309748+0000","last_undegraded":"2023-03-25T04:29:33.309748+0000","last_fullsized":"2023-03-25T04:29:33.309748+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:33.309729+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:29:33.309729+0000","last_peered":"2023-03-25T04:29:33.309729+0000","last_clean":"2023-03-25T04:29:33.309729+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:29:33.309729+0000","last_undegraded":"2023-03-25T04:29:33.309729+0000","last_fullsized":"2023-03-25T04:29:33.309729+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:23.308607+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:29:23.308607+0000","last_peered":"2023-03-25T04:29:23.308607+0000","last_clean":"2023-03-25T04:29:23.308607+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:29:23.308607+0000","last_undegraded":"2023-03-25T04:29:23.308607+0000","last_fullsized":"2023-03-25T04:29:23.308607+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":28,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:20.554710+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:29:20.554710+0000","last_peered":"2023-03-25T04:29:20.554710+0000","last_clean":"2023-03-25T04:29:20.554710+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:29:20.554710+0000","last_undegraded":"2023-03-25T04:29:20.554710+0000","last_fullsized":"2023-03-25T04:29:20.554710+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309216+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:29:28.309216+0000","last_peered":"2023-03-25T04:29:28.309216+0000","last_clean":"2023-03-25T04:29:28.309216+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:29:28.309216+0000","last_undegraded":"2023-03-25T04:29:28.309216+0000","last_fullsized":"2023-03-25T04:29:28.309216+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309233+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:29:28.309233+0000","last_peered":"2023-03-25T04:29:28.309233+0000","last_clean":"2023-03-25T04:29:28.309233+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:29:28.309233+0000","last_undegraded":"2023-03-25T04:29:28.309233+0000","last_fullsized":"2023-03-25T04:29:28.309233+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117058,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:31:42 2023","interfaces":[{"interface":"back","average":{"1min":0.27500000000000002,"5min":0.28399999999999997,"15min":0.29399999999999998},"min":{"1min":0.19800000000000001,"5min":0.191,"15min":0.191},"max":{"1min":0.36899999999999999,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.30599999999999999},{"interface":"front","average":{"1min":0.27900000000000003,"5min":0.28499999999999998,"15min":0.29599999999999999},"min":{"1min":0.216,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":0.40799999999999997,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.32500000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051649,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:31:35 2023","interfaces":[{"interface":"back","average":{"1min":0.26100000000000001,"5min":0.25700000000000001,"15min":0.251},"min":{"1min":0.19,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.35899999999999999,"5min":0.41599999999999998,"15min":0.41599999999999998},"last":0.24199999999999999},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.27400000000000002,"15min":0.26700000000000002},"min":{"1min":0.20200000000000001,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.374,"5min":0.42899999999999999,"15min":0.42899999999999999},"last":0.22600000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:32:54.944+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:32:55.469+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:32:55.927+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)

Ignoring malparsed backtrace: 2023-03-25T04:33:13.958 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean
, 2023-03-25T04:33:13.959 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}
, 2023-03-25T04:33:13.959 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json

Ignoring malparsed backtrace: 2023-03-25T04:33:31.199 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":267,"stamp":"2023-03-25T04:33:30.166660+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002269"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:33.309748+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:29:33.309748+0000","last_peered":"2023-03-25T04:29:33.309748+0000","last_clean":"2023-03-25T04:29:33.309748+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:29:33.309748+0000","last_undegraded":"2023-03-25T04:29:33.309748+0000","last_fullsized":"2023-03-25T04:29:33.309748+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:33.309729+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:29:33.309729+0000","last_peered":"2023-03-25T04:29:33.309729+0000","last_clean":"2023-03-25T04:29:33.309729+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:29:33.309729+0000","last_undegraded":"2023-03-25T04:29:33.309729+0000","last_fullsized":"2023-03-25T04:29:33.309729+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":28,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:31:23.320894+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:31:23.320894+0000","last_peered":"2023-03-25T04:31:23.320894+0000","last_clean":"2023-03-25T04:31:23.320894+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:31:23.320894+0000","last_undegraded":"2023-03-25T04:31:23.320894+0000","last_fullsized":"2023-03-25T04:31:23.320894+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":28,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:20.554710+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:29:20.554710+0000","last_peered":"2023-03-25T04:29:20.554710+0000","last_clean":"2023-03-25T04:29:20.554710+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:29:20.554710+0000","last_undegraded":"2023-03-25T04:29:20.554710+0000","last_fullsized":"2023-03-25T04:29:20.554710+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309216+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:29:28.309216+0000","last_peered":"2023-03-25T04:29:28.309216+0000","last_clean":"2023-03-25T04:29:28.309216+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:29:28.309216+0000","last_undegraded":"2023-03-25T04:29:28.309216+0000","last_fullsized":"2023-03-25T04:29:28.309216+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309233+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:29:28.309233+0000","last_peered":"2023-03-25T04:29:28.309233+0000","last_clean":"2023-03-25T04:29:28.309233+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:29:28.309233+0000","last_undegraded":"2023-03-25T04:29:28.309233+0000","last_fullsized":"2023-03-25T04:29:28.309233+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117068,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:32:46 2023","interfaces":[{"interface":"back","average":{"1min":0.27000000000000002,"5min":0.27900000000000003,"15min":0.29199999999999998},"min":{"1min":0.17899999999999999,"5min":0.17899999999999999,"15min":0.17899999999999999},"max":{"1min":0.35599999999999998,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.28799999999999998},{"interface":"front","average":{"1min":0.26800000000000002,"5min":0.27800000000000002,"15min":0.29399999999999998},"min":{"1min":0.19900000000000001,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":0.377,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.315}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051660,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:32:36 2023","interfaces":[{"interface":"back","average":{"1min":0.248,"5min":0.25700000000000001,"15min":0.251},"min":{"1min":0.17000000000000001,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"last":0.22700000000000001},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.27600000000000002,"15min":0.26800000000000002},"min":{"1min":0.192,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"last":0.20399999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:33:31.201 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:33:31.400+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)

Ignoring malparsed backtrace: 2023-03-25T04:33:47.531+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:33:47.951+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:33:48.505+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)

Ignoring malparsed backtrace: 2023-03-25T04:34:06.104 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json
, 2023-03-25T04:34:06.118+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:34:06.497 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:34:23.399 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:34:23.665+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:34:23.712 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:34:40.195+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:34:40.523+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:34:40.676 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace: 2023-03-25T04:35:03.381+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:35:04.139+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:35:04.391+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)

Ignoring malparsed backtrace: 2023-03-25T04:35:22.738 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean
, 2023-03-25T04:35:22.739 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}
, 2023-03-25T04:35:22.739 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json

Ignoring malparsed backtrace: 2023-03-25T04:35:41.393+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:35:42.352+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:35:42.376+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:35:57.384 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":340,"stamp":"2023-03-25T04:35:56.194992+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.001930"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":36,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:31:43.322894+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:31:43.322894+0000","last_peered":"2023-03-25T04:31:43.322894+0000","last_clean":"2023-03-25T04:31:43.322894+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:31:43.322894+0000","last_undegraded":"2023-03-25T04:31:43.322894+0000","last_fullsized":"2023-03-25T04:31:43.322894+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":28,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:31:43.322876+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:31:43.322876+0000","last_peered":"2023-03-25T04:31:43.322876+0000","last_clean":"2023-03-25T04:31:43.322876+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:31:43.322876+0000","last_undegraded":"2023-03-25T04:31:43.322876+0000","last_fullsized":"2023-03-25T04:31:43.322876+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:23.331998+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:33:23.331998+0000","last_peered":"2023-03-25T04:33:23.331998+0000","last_clean":"2023-03-25T04:33:23.331998+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:33:23.331998+0000","last_undegraded":"2023-03-25T04:33:23.331998+0000","last_fullsized":"2023-03-25T04:33:23.331998+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:30.576723+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:33:30.576723+0000","last_peered":"2023-03-25T04:33:30.576723+0000","last_clean":"2023-03-25T04:33:30.576723+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:33:30.576723+0000","last_undegraded":"2023-03-25T04:33:30.576723+0000","last_fullsized":"2023-03-25T04:33:30.576723+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":37,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:38.333447+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:33:38.333447+0000","last_peered":"2023-03-25T04:33:38.333447+0000","last_clean":"2023-03-25T04:33:38.333447+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:33:38.333447+0000","last_undegraded":"2023-03-25T04:33:38.333447+0000","last_fullsized":"2023-03-25T04:33:38.333447+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:38.333467+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:33:38.333467+0000","last_peered":"2023-03-25T04:33:38.333467+0000","last_clean":"2023-03-25T04:33:38.333467+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:33:38.333467+0000","last_undegraded":"2023-03-25T04:33:38.333467+0000","last_fullsized":"2023-03-25T04:33:38.333467+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117098,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:35:49 2023","interfaces":[{"interface":"back","average":{"1min":0.255,"5min":0.26000000000000001,"15min":0.28199999999999997},"min":{"1min":0.17899999999999999,"5min":0.16,"15min":0.16},"max":{"1min":0.36399999999999999,"5min":0.42599999999999999,"15min":0.45100000000000001},"last":0.26100000000000001},{"interface":"front","average":{"1min":0.26000000000000001,"5min":0.26700000000000002,"15min":0.28599999999999998},"min":{"1min":0.188,"5min":0.17999999999999999,"15min":0.17499999999999999},"max":{"1min":0.39100000000000001,"5min":0.45200000000000001,"15min":0.46300000000000002},"last":0.28399999999999997}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051689,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:35:42 2023","interfaces":[{"interface":"back","average":{"1min":0.29399999999999998,"5min":0.27200000000000002,"15min":0.25800000000000001},"min":{"1min":0.17299999999999999,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.41099999999999998,"5min":0.42699999999999999,"15min":0.42699999999999999},"last":0.20399999999999999},{"interface":"front","average":{"1min":0.30299999999999999,"5min":0.28399999999999997,"15min":0.27300000000000002},"min":{"1min":0.19700000000000001,"5min":0.192,"15min":0.16600000000000001},"max":{"1min":0.43099999999999999,"5min":0.45800000000000002,"15min":0.45800000000000002},"last":0.22}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:35:57.624+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)

Ignoring malparsed backtrace: 2023-03-25T04:36:17.226 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:36:17.239 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":350,"stamp":"2023-03-25T04:36:16.198866+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002148"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":37,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:53.335026+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:33:53.335026+0000","last_peered":"2023-03-25T04:33:53.335026+0000","last_clean":"2023-03-25T04:33:53.335026+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:33:53.335026+0000","last_undegraded":"2023-03-25T04:33:53.335026+0000","last_fullsized":"2023-03-25T04:33:53.335026+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:53.335009+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:33:53.335009+0000","last_peered":"2023-03-25T04:33:53.335009+0000","last_clean":"2023-03-25T04:33:53.335009+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:33:53.335009+0000","last_undegraded":"2023-03-25T04:33:53.335009+0000","last_fullsized":"2023-03-25T04:33:53.335009+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:23.331998+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:33:23.331998+0000","last_peered":"2023-03-25T04:33:23.331998+0000","last_clean":"2023-03-25T04:33:23.331998+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:33:23.331998+0000","last_undegraded":"2023-03-25T04:33:23.331998+0000","last_fullsized":"2023-03-25T04:33:23.331998+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:30.576723+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:33:30.576723+0000","last_peered":"2023-03-25T04:33:30.576723+0000","last_clean":"2023-03-25T04:33:30.576723+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:33:30.576723+0000","last_undegraded":"2023-03-25T04:33:30.576723+0000","last_fullsized":"2023-03-25T04:33:30.576723+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":37,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:38.333447+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:33:38.333447+0000","last_peered":"2023-03-25T04:33:38.333447+0000","last_clean":"2023-03-25T04:33:38.333447+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:33:38.333447+0000","last_undegraded":"2023-03-25T04:33:38.333447+0000","last_fullsized":"2023-03-25T04:33:38.333447+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:38.333467+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:33:38.333467+0000","last_peered":"2023-03-25T04:33:38.333467+0000","last_clean":"2023-03-25T04:33:38.333467+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:33:38.333467+0000","last_undegraded":"2023-03-25T04:33:38.333467+0000","last_fullsized":"2023-03-25T04:33:38.333467+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117102,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:35:49 2023","interfaces":[{"interface":"back","average":{"1min":0.255,"5min":0.26000000000000001,"15min":0.28199999999999997},"min":{"1min":0.17899999999999999,"5min":0.16,"15min":0.16},"max":{"1min":0.36399999999999999,"5min":0.42599999999999999,"15min":0.45100000000000001},"last":0.28399999999999997},{"interface":"front","average":{"1min":0.26000000000000001,"5min":0.26700000000000002,"15min":0.28599999999999998},"min":{"1min":0.188,"5min":0.17999999999999999,"15min":0.17499999999999999},"max":{"1min":0.39100000000000001,"5min":0.45200000000000001,"15min":0.46300000000000002},"last":0.30299999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051693,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:35:42 2023","interfaces":[{"interface":"back","average":{"1min":0.29399999999999998,"5min":0.27200000000000002,"15min":0.25800000000000001},"min":{"1min":0.17299999999999999,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.41099999999999998,"5min":0.42699999999999999,"15min":0.42699999999999999},"last":0.35099999999999998},{"interface":"front","average":{"1min":0.30299999999999999,"5min":0.28399999999999997,"15min":0.27300000000000002},"min":{"1min":0.19700000000000001,"5min":0.192,"15min":0.16600000000000001},"max":{"1min":0.43099999999999999,"5min":0.45800000000000002,"15min":0.45800000000000002},"last":0.32900000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:36:34.556+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:36:34.717+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:36:35.583+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)

Ignoring malparsed backtrace: 2023-03-25T04:36:52.680 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json
, 2023-03-25T04:36:52.701+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:36:53.067 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:37:09.963 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:37:10.273 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all

Ignoring malparsed backtrace: 2023-03-25T04:37:26.856+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:37:27.288 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:37:27.538+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:37:41.903 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":392,"stamp":"2023-03-25T04:37:40.213373+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002012"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":37,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:53.335026+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:33:53.335026+0000","last_peered":"2023-03-25T04:33:53.335026+0000","last_clean":"2023-03-25T04:33:53.335026+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:33:53.335026+0000","last_undegraded":"2023-03-25T04:33:53.335026+0000","last_fullsized":"2023-03-25T04:33:53.335026+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:53.335009+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:33:53.335009+0000","last_peered":"2023-03-25T04:33:53.335009+0000","last_clean":"2023-03-25T04:33:53.335009+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:33:53.335009+0000","last_undegraded":"2023-03-25T04:33:53.335009+0000","last_fullsized":"2023-03-25T04:33:53.335009+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:23.343585+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:35:23.343585+0000","last_peered":"2023-03-25T04:35:23.343585+0000","last_clean":"2023-03-25T04:35:23.343585+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:35:23.343585+0000","last_undegraded":"2023-03-25T04:35:23.343585+0000","last_fullsized":"2023-03-25T04:35:23.343585+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:30.576723+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:33:30.576723+0000","last_peered":"2023-03-25T04:33:30.576723+0000","last_clean":"2023-03-25T04:33:30.576723+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:33:30.576723+0000","last_undegraded":"2023-03-25T04:33:30.576723+0000","last_fullsized":"2023-03-25T04:33:30.576723+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":37,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:38.333447+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:33:38.333447+0000","last_peered":"2023-03-25T04:33:38.333447+0000","last_clean":"2023-03-25T04:33:38.333447+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:33:38.333447+0000","last_undegraded":"2023-03-25T04:33:38.333447+0000","last_fullsized":"2023-03-25T04:33:38.333447+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:38.333467+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:33:38.333467+0000","last_peered":"2023-03-25T04:33:38.333467+0000","last_clean":"2023-03-25T04:33:38.333467+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:33:38.333467+0000","last_undegraded":"2023-03-25T04:33:38.333467+0000","last_fullsized":"2023-03-25T04:33:38.333467+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117119,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:36:50 2023","interfaces":[{"interface":"back","average":{"1min":0.254,"5min":0.25600000000000001,"15min":0.27900000000000003},"min":{"1min":0.20499999999999999,"5min":0.16,"15min":0.16},"max":{"1min":0.371,"5min":0.42599999999999999,"15min":0.45100000000000001},"last":0.23000000000000001},{"interface":"front","average":{"1min":0.26800000000000002,"5min":0.26500000000000001,"15min":0.28399999999999997},"min":{"1min":0.19700000000000001,"5min":0.17999999999999999,"15min":0.17499999999999999},"max":{"1min":0.34699999999999998,"5min":0.45200000000000001,"15min":0.46300000000000002},"last":0.20899999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051710,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:36:44 2023","interfaces":[{"interface":"back","average":{"1min":0.28399999999999997,"5min":0.27700000000000002,"15min":0.26100000000000001},"min":{"1min":0.192,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.40899999999999997,"5min":0.42699999999999999,"15min":0.42699999999999999},"last":0.39700000000000002},{"interface":"front","average":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.27500000000000002},"min":{"1min":0.182,"5min":0.182,"15min":0.16600000000000001},"max":{"1min":0.439,"5min":0.45800000000000002,"15min":0.45800000000000002},"last":0.439}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:37:42.400+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)

Ignoring malparsed backtrace: 2023-03-25T04:38:04.766+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:38:04.922 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:38:05.246 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:38:19.605 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":411,"stamp":"2023-03-25T04:38:18.220966+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002368"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":38,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:36:03.347070+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:36:03.347070+0000","last_peered":"2023-03-25T04:36:03.347070+0000","last_clean":"2023-03-25T04:36:03.347070+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:36:03.347070+0000","last_undegraded":"2023-03-25T04:36:03.347070+0000","last_fullsized":"2023-03-25T04:36:03.347070+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:36:03.347025+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:36:03.347025+0000","last_peered":"2023-03-25T04:36:03.347025+0000","last_clean":"2023-03-25T04:36:03.347025+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:36:03.347025+0000","last_undegraded":"2023-03-25T04:36:03.347025+0000","last_fullsized":"2023-03-25T04:36:03.347025+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:23.343585+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:35:23.343585+0000","last_peered":"2023-03-25T04:35:23.343585+0000","last_clean":"2023-03-25T04:35:23.343585+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:35:23.343585+0000","last_undegraded":"2023-03-25T04:35:23.343585+0000","last_fullsized":"2023-03-25T04:35:23.343585+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:35.587025+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:35:35.587025+0000","last_peered":"2023-03-25T04:35:35.587025+0000","last_clean":"2023-03-25T04:35:35.587025+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:35:35.587025+0000","last_undegraded":"2023-03-25T04:35:35.587025+0000","last_fullsized":"2023-03-25T04:35:35.587025+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":38,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:43.345308+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:35:43.345308+0000","last_peered":"2023-03-25T04:35:43.345308+0000","last_clean":"2023-03-25T04:35:43.345308+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:35:43.345308+0000","last_undegraded":"2023-03-25T04:35:43.345308+0000","last_fullsized":"2023-03-25T04:35:43.345308+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:43.345324+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:35:43.345324+0000","last_peered":"2023-03-25T04:35:43.345324+0000","last_clean":"2023-03-25T04:35:43.345324+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:35:43.345324+0000","last_undegraded":"2023-03-25T04:35:43.345324+0000","last_fullsized":"2023-03-25T04:35:43.345324+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117127,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:37:53 2023","interfaces":[{"interface":"back","average":{"1min":0.25800000000000001,"5min":0.253,"15min":0.27700000000000002},"min":{"1min":0.184,"5min":0.16,"15min":0.16},"max":{"1min":0.44600000000000001,"5min":0.44600000000000001,"15min":0.45100000000000001},"last":0.35199999999999998},{"interface":"front","average":{"1min":0.26900000000000002,"5min":0.26500000000000001,"15min":0.28199999999999997},"min":{"1min":0.192,"5min":0.17999999999999999,"15min":0.17499999999999999},"max":{"1min":0.46400000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"last":0.33200000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051717,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:37:44 2023","interfaces":[{"interface":"back","average":{"1min":0.27300000000000002,"5min":0.28199999999999997,"15min":0.26200000000000001},"min":{"1min":0.17899999999999999,"5min":0.17299999999999999,"15min":0.17000000000000001},"max":{"1min":0.41799999999999998,"5min":0.42099999999999999,"15min":0.42699999999999999},"last":0.28699999999999998},{"interface":"front","average":{"1min":0.29399999999999998,"5min":0.29099999999999998,"15min":0.27700000000000002},"min":{"1min":0.19900000000000001,"5min":0.182,"15min":0.16600000000000001},"max":{"1min":0.441,"5min":0.441,"15min":0.45800000000000002},"last":0.26300000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:38:19.718+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:38:20.315+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)

Ignoring malparsed backtrace: 2023-03-25T04:38:44.283 INFO:teuthology.orchestra.run.smithi070.stdout:
, 2023-03-25T04:38:44.283 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":781,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":16,"modified":"2023-03-25T04:38:40.225321+0000","services":{}},"progress_events":{}}
, 2023-03-25T04:38:44.296 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now

Ignoring malparsed backtrace: 2023-03-25T04:39:02.477+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:39:02.578+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:39:03.524+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:39:20.359 INFO:teuthology.orchestra.run.smithi070.stdout:
, 2023-03-25T04:39:20.359 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":818,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":16,"modified":"2023-03-25T04:38:40.225321+0000","services":{}},"progress_events":{}}
, 2023-03-25T04:39:20.372 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now

Ignoring malparsed backtrace: 2023-03-25T04:39:37.527 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:39:37.541 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":450,"stamp":"2023-03-25T04:39:36.235332+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002086"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":38,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:36:03.347070+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:36:03.347070+0000","last_peered":"2023-03-25T04:36:03.347070+0000","last_clean":"2023-03-25T04:36:03.347070+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:36:03.347070+0000","last_undegraded":"2023-03-25T04:36:03.347070+0000","last_fullsized":"2023-03-25T04:36:03.347070+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:36:03.347025+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:36:03.347025+0000","last_peered":"2023-03-25T04:36:03.347025+0000","last_clean":"2023-03-25T04:36:03.347025+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:36:03.347025+0000","last_undegraded":"2023-03-25T04:36:03.347025+0000","last_fullsized":"2023-03-25T04:36:03.347025+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:23.353778+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:37:23.353778+0000","last_peered":"2023-03-25T04:37:23.353778+0000","last_clean":"2023-03-25T04:37:23.353778+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:37:23.353778+0000","last_undegraded":"2023-03-25T04:37:23.353778+0000","last_fullsized":"2023-03-25T04:37:23.353778+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:35.587025+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:35:35.587025+0000","last_peered":"2023-03-25T04:35:35.587025+0000","last_clean":"2023-03-25T04:35:35.587025+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:35:35.587025+0000","last_undegraded":"2023-03-25T04:35:35.587025+0000","last_fullsized":"2023-03-25T04:35:35.587025+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":38,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:43.345308+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:35:43.345308+0000","last_peered":"2023-03-25T04:35:43.345308+0000","last_clean":"2023-03-25T04:35:43.345308+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:35:43.345308+0000","last_undegraded":"2023-03-25T04:35:43.345308+0000","last_fullsized":"2023-03-25T04:35:43.345308+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:43.345324+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:35:43.345324+0000","last_peered":"2023-03-25T04:35:43.345324+0000","last_clean":"2023-03-25T04:35:43.345324+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:35:43.345324+0000","last_undegraded":"2023-03-25T04:35:43.345324+0000","last_fullsized":"2023-03-25T04:35:43.345324+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117143,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:38:54 2023","interfaces":[{"interface":"back","average":{"1min":0.25800000000000001,"5min":0.25700000000000001,"15min":0.27400000000000002},"min":{"1min":0.192,"5min":0.17399999999999999,"15min":0.16},"max":{"1min":0.39400000000000002,"5min":0.44600000000000001,"15min":0.45100000000000001},"last":0.26500000000000001},{"interface":"front","average":{"1min":0.25900000000000001,"5min":0.26700000000000002,"15min":0.27900000000000003},"min":{"1min":0.19500000000000001,"5min":0.188,"15min":0.17499999999999999},"max":{"1min":0.41499999999999998,"5min":0.46400000000000002,"15min":0.46400000000000002},"last":0.29099999999999998}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051733,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:38:45 2023","interfaces":[{"interface":"back","average":{"1min":0.27300000000000002,"5min":0.28100000000000003,"15min":0.26400000000000001},"min":{"1min":0.19400000000000001,"5min":0.17299999999999999,"15min":0.17000000000000001},"max":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"last":0.27300000000000002},{"interface":"front","average":{"1min":0.28799999999999998,"5min":0.29099999999999998,"15min":0.27800000000000002},"min":{"1min":0.19800000000000001,"5min":0.182,"15min":0.16600000000000001},"max":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"last":0.308}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:39:54.416+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:39:54.455 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:39:54.780 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:40:09.288+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:40:09.389+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:40:10.267+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)

Ignoring malparsed backtrace: 2023-03-25T04:40:28.984 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":476,"stamp":"2023-03-25T04:40:28.245911+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002693"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":39,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:38:13.358968+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:38:13.358968+0000","last_peered":"2023-03-25T04:38:13.358968+0000","last_clean":"2023-03-25T04:38:13.358968+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:38:13.358968+0000","last_undegraded":"2023-03-25T04:38:13.358968+0000","last_fullsized":"2023-03-25T04:38:13.358968+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:38:13.358922+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:38:13.358922+0000","last_peered":"2023-03-25T04:38:13.358922+0000","last_clean":"2023-03-25T04:38:13.358922+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:38:13.358922+0000","last_undegraded":"2023-03-25T04:38:13.358922+0000","last_fullsized":"2023-03-25T04:38:13.358922+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:23.353778+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:37:23.353778+0000","last_peered":"2023-03-25T04:37:23.353778+0000","last_clean":"2023-03-25T04:37:23.353778+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:37:23.353778+0000","last_undegraded":"2023-03-25T04:37:23.353778+0000","last_fullsized":"2023-03-25T04:37:23.353778+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:40.598579+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:37:40.598579+0000","last_peered":"2023-03-25T04:37:40.598579+0000","last_clean":"2023-03-25T04:37:40.598579+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:37:40.598579+0000","last_undegraded":"2023-03-25T04:37:40.598579+0000","last_fullsized":"2023-03-25T04:37:40.598579+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":39,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:48.356344+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:37:48.356344+0000","last_peered":"2023-03-25T04:37:48.356344+0000","last_clean":"2023-03-25T04:37:48.356344+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:37:48.356344+0000","last_undegraded":"2023-03-25T04:37:48.356344+0000","last_fullsized":"2023-03-25T04:37:48.356344+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:48.356365+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:37:48.356365+0000","last_peered":"2023-03-25T04:37:48.356365+0000","last_clean":"2023-03-25T04:37:48.356365+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:37:48.356365+0000","last_undegraded":"2023-03-25T04:37:48.356365+0000","last_fullsized":"2023-03-25T04:37:48.356365+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117153,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:39:54 2023","interfaces":[{"interface":"back","average":{"1min":0.25900000000000001,"5min":0.25600000000000001,"15min":0.27100000000000002},"min":{"1min":0.188,"5min":0.17899999999999999,"15min":0.16},"max":{"1min":0.41999999999999998,"5min":0.44600000000000001,"15min":0.45100000000000001},"last":0.255},{"interface":"front","average":{"1min":0.27500000000000002,"5min":0.26600000000000001,"15min":0.27700000000000002},"min":{"1min":0.17699999999999999,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.45600000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"last":0.26800000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051743,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:39:47 2023","interfaces":[{"interface":"back","average":{"1min":0.26300000000000001,"5min":0.27700000000000002,"15min":0.26500000000000001},"min":{"1min":0.187,"5min":0.17299999999999999,"15min":0.17000000000000001},"max":{"1min":0.433,"5min":0.433,"15min":0.433},"last":0.36599999999999999},{"interface":"front","average":{"1min":0.27100000000000002,"5min":0.28799999999999998,"15min":0.27900000000000003},"min":{"1min":0.20100000000000001,"5min":0.182,"15min":0.16600000000000001},"max":{"1min":0.48599999999999999,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.34999999999999998}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:40:28.986 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean
, 2023-03-25T04:40:28.986 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}

Ignoring malparsed backtrace: 2023-03-25T04:40:47.229+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:40:47.503+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:40:48.251+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)

Ignoring malparsed backtrace: 2023-03-25T04:41:03.267 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:41:03.578 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all

Ignoring malparsed backtrace: 2023-03-25T04:41:21.347 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":502,"stamp":"2023-03-25T04:41:20.256204+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002342"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":39,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:38:13.358968+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:38:13.358968+0000","last_peered":"2023-03-25T04:38:13.358968+0000","last_clean":"2023-03-25T04:38:13.358968+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:38:13.358968+0000","last_undegraded":"2023-03-25T04:38:13.358968+0000","last_fullsized":"2023-03-25T04:38:13.358968+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:38:13.358922+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:38:13.358922+0000","last_peered":"2023-03-25T04:38:13.358922+0000","last_clean":"2023-03-25T04:38:13.358922+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:38:13.358922+0000","last_undegraded":"2023-03-25T04:38:13.358922+0000","last_fullsized":"2023-03-25T04:38:13.358922+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:23.353778+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:37:23.353778+0000","last_peered":"2023-03-25T04:37:23.353778+0000","last_clean":"2023-03-25T04:37:23.353778+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:37:23.353778+0000","last_undegraded":"2023-03-25T04:37:23.353778+0000","last_fullsized":"2023-03-25T04:37:23.353778+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:40.598579+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:37:40.598579+0000","last_peered":"2023-03-25T04:37:40.598579+0000","last_clean":"2023-03-25T04:37:40.598579+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:37:40.598579+0000","last_undegraded":"2023-03-25T04:37:40.598579+0000","last_fullsized":"2023-03-25T04:37:40.598579+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":39,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:48.356344+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:37:48.356344+0000","last_peered":"2023-03-25T04:37:48.356344+0000","last_clean":"2023-03-25T04:37:48.356344+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:37:48.356344+0000","last_undegraded":"2023-03-25T04:37:48.356344+0000","last_fullsized":"2023-03-25T04:37:48.356344+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:48.356365+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:37:48.356365+0000","last_peered":"2023-03-25T04:37:48.356365+0000","last_clean":"2023-03-25T04:37:48.356365+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:37:48.356365+0000","last_undegraded":"2023-03-25T04:37:48.356365+0000","last_fullsized":"2023-03-25T04:37:48.356365+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117163,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:40:56 2023","interfaces":[{"interface":"back","average":{"1min":0.28799999999999998,"5min":0.26300000000000001,"15min":0.27100000000000002},"min":{"1min":0.188,"5min":0.184,"15min":0.16},"max":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.17499999999999999},{"interface":"front","average":{"1min":0.29599999999999999,"5min":0.27300000000000002,"15min":0.27700000000000002},"min":{"1min":0.20599999999999999,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.43099999999999999,"5min":0.46400000000000002,"15min":0.46400000000000002},"last":0.20699999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051754,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:40:52 2023","interfaces":[{"interface":"back","average":{"1min":0.28599999999999998,"5min":0.27500000000000002,"15min":0.26700000000000002},"min":{"1min":0.17899999999999999,"5min":0.17899999999999999,"15min":0.17000000000000001},"max":{"1min":0.44,"5min":0.44,"15min":0.44},"last":0.23699999999999999},{"interface":"front","average":{"1min":0.29799999999999999,"5min":0.28699999999999998,"15min":0.28100000000000003},"min":{"1min":0.20300000000000001,"5min":0.182,"15min":0.16600000000000001},"max":{"1min":0.47599999999999998,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.28000000000000003}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:41:21.349 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean
, 2023-03-25T04:41:21.349 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}

Ignoring malparsed backtrace: 2023-03-25T04:41:39.867+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:41:40.383+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:41:40.905+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:41:58.219 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json
, 2023-03-25T04:41:58.319+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:41:58.644 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:42:18.193 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":530,"stamp":"2023-03-25T04:42:16.266659+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002356"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":39,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:38:13.358968+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:38:13.358968+0000","last_peered":"2023-03-25T04:38:13.358968+0000","last_clean":"2023-03-25T04:38:13.358968+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:38:13.358968+0000","last_undegraded":"2023-03-25T04:38:13.358968+0000","last_fullsized":"2023-03-25T04:38:13.358968+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:38:13.358922+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:38:13.358922+0000","last_peered":"2023-03-25T04:38:13.358922+0000","last_clean":"2023-03-25T04:38:13.358922+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:38:13.358922+0000","last_undegraded":"2023-03-25T04:38:13.358922+0000","last_fullsized":"2023-03-25T04:38:13.358922+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:23.364593+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:39:23.364593+0000","last_peered":"2023-03-25T04:39:23.364593+0000","last_clean":"2023-03-25T04:39:23.364593+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:39:23.364593+0000","last_undegraded":"2023-03-25T04:39:23.364593+0000","last_fullsized":"2023-03-25T04:39:23.364593+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:45.608475+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:39:45.608475+0000","last_peered":"2023-03-25T04:39:45.608475+0000","last_clean":"2023-03-25T04:39:45.608475+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:39:45.608475+0000","last_undegraded":"2023-03-25T04:39:45.608475+0000","last_fullsized":"2023-03-25T04:39:45.608475+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":40,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:53.367398+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:39:53.367398+0000","last_peered":"2023-03-25T04:39:53.367398+0000","last_clean":"2023-03-25T04:39:53.367398+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:39:53.367398+0000","last_undegraded":"2023-03-25T04:39:53.367398+0000","last_fullsized":"2023-03-25T04:39:53.367398+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:53.367424+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:39:53.367424+0000","last_peered":"2023-03-25T04:39:53.367424+0000","last_clean":"2023-03-25T04:39:53.367424+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:39:53.367424+0000","last_undegraded":"2023-03-25T04:39:53.367424+0000","last_fullsized":"2023-03-25T04:39:53.367424+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117175,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:41:58 2023","interfaces":[{"interface":"back","average":{"1min":0.25800000000000001,"5min":0.26400000000000001,"15min":0.26800000000000002},"min":{"1min":0.17499999999999999,"5min":0.17499999999999999,"15min":0.16},"max":{"1min":0.39100000000000001,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.22900000000000001},{"interface":"front","average":{"1min":0.26300000000000001,"5min":0.27200000000000002,"15min":0.27400000000000002},"min":{"1min":0.20599999999999999,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.35699999999999998,"5min":0.46400000000000002,"15min":0.46400000000000002},"last":0.26400000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051765,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:41:52 2023","interfaces":[{"interface":"back","average":{"1min":0.26500000000000001,"5min":0.27200000000000002,"15min":0.26800000000000002},"min":{"1min":0.17000000000000001,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.39200000000000002,"5min":0.44,"15min":0.44},"last":0.23899999999999999},{"interface":"front","average":{"1min":0.28100000000000003,"5min":0.28599999999999998,"15min":0.28199999999999997},"min":{"1min":0.19400000000000001,"5min":0.19400000000000001,"15min":0.16600000000000001},"max":{"1min":0.41399999999999998,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.254}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:42:18.297+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:42:18.958+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:42:35.512 INFO:teuthology.orchestra.run.smithi070.stdout:
, 2023-03-25T04:42:35.512 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":1013,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":17,"modified":"2023-03-25T04:41:54.262744+0000","services":{}},"progress_events":{}}
, 2023-03-25T04:42:35.525 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:42:52.741 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":548,"stamp":"2023-03-25T04:42:52.273601+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002221"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":40,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:40:23.370056+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:40:23.370056+0000","last_peered":"2023-03-25T04:40:23.370056+0000","last_clean":"2023-03-25T04:40:23.370056+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:40:23.370056+0000","last_undegraded":"2023-03-25T04:40:23.370056+0000","last_fullsized":"2023-03-25T04:40:23.370056+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:40:23.370039+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:40:23.370039+0000","last_peered":"2023-03-25T04:40:23.370039+0000","last_clean":"2023-03-25T04:40:23.370039+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:40:23.370039+0000","last_undegraded":"2023-03-25T04:40:23.370039+0000","last_fullsized":"2023-03-25T04:40:23.370039+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:23.364593+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:39:23.364593+0000","last_peered":"2023-03-25T04:39:23.364593+0000","last_clean":"2023-03-25T04:39:23.364593+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:39:23.364593+0000","last_undegraded":"2023-03-25T04:39:23.364593+0000","last_fullsized":"2023-03-25T04:39:23.364593+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:45.608475+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:39:45.608475+0000","last_peered":"2023-03-25T04:39:45.608475+0000","last_clean":"2023-03-25T04:39:45.608475+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:39:45.608475+0000","last_undegraded":"2023-03-25T04:39:45.608475+0000","last_fullsized":"2023-03-25T04:39:45.608475+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":40,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:53.367398+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:39:53.367398+0000","last_peered":"2023-03-25T04:39:53.367398+0000","last_clean":"2023-03-25T04:39:53.367398+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:39:53.367398+0000","last_undegraded":"2023-03-25T04:39:53.367398+0000","last_fullsized":"2023-03-25T04:39:53.367398+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:53.367424+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:39:53.367424+0000","last_peered":"2023-03-25T04:39:53.367424+0000","last_clean":"2023-03-25T04:39:53.367424+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:39:53.367424+0000","last_undegraded":"2023-03-25T04:39:53.367424+0000","last_fullsized":"2023-03-25T04:39:53.367424+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117182,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:41:58 2023","interfaces":[{"interface":"back","average":{"1min":0.25800000000000001,"5min":0.26400000000000001,"15min":0.26800000000000002},"min":{"1min":0.17499999999999999,"5min":0.17499999999999999,"15min":0.16},"max":{"1min":0.39100000000000001,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.19600000000000001},{"interface":"front","average":{"1min":0.26300000000000001,"5min":0.27200000000000002,"15min":0.27400000000000002},"min":{"1min":0.20599999999999999,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.35699999999999998,"5min":0.46400000000000002,"15min":0.46400000000000002},"last":0.219}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051773,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:41:52 2023","interfaces":[{"interface":"back","average":{"1min":0.26500000000000001,"5min":0.27200000000000002,"15min":0.26800000000000002},"min":{"1min":0.17000000000000001,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.39200000000000002,"5min":0.44,"15min":0.44},"last":0.19},{"interface":"front","average":{"1min":0.28100000000000003,"5min":0.28599999999999998,"15min":0.28199999999999997},"min":{"1min":0.19400000000000001,"5min":0.19400000000000001,"15min":0.16600000000000001},"max":{"1min":0.41399999999999998,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.17599999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:42:52.742 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean

Ignoring malparsed backtrace: 2023-03-25T04:43:09.973 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:43:09.986 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":556,"stamp":"2023-03-25T04:43:08.276556+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002359"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":40,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:40:23.370056+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:40:23.370056+0000","last_peered":"2023-03-25T04:40:23.370056+0000","last_clean":"2023-03-25T04:40:23.370056+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:40:23.370056+0000","last_undegraded":"2023-03-25T04:40:23.370056+0000","last_fullsized":"2023-03-25T04:40:23.370056+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:40:23.370039+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:40:23.370039+0000","last_peered":"2023-03-25T04:40:23.370039+0000","last_clean":"2023-03-25T04:40:23.370039+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:40:23.370039+0000","last_undegraded":"2023-03-25T04:40:23.370039+0000","last_fullsized":"2023-03-25T04:40:23.370039+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:23.364593+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:39:23.364593+0000","last_peered":"2023-03-25T04:39:23.364593+0000","last_clean":"2023-03-25T04:39:23.364593+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:39:23.364593+0000","last_undegraded":"2023-03-25T04:39:23.364593+0000","last_fullsized":"2023-03-25T04:39:23.364593+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:45.608475+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:39:45.608475+0000","last_peered":"2023-03-25T04:39:45.608475+0000","last_clean":"2023-03-25T04:39:45.608475+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:39:45.608475+0000","last_undegraded":"2023-03-25T04:39:45.608475+0000","last_fullsized":"2023-03-25T04:39:45.608475+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":40,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:53.367398+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:39:53.367398+0000","last_peered":"2023-03-25T04:39:53.367398+0000","last_clean":"2023-03-25T04:39:53.367398+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:39:53.367398+0000","last_undegraded":"2023-03-25T04:39:53.367398+0000","last_fullsized":"2023-03-25T04:39:53.367398+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:53.367424+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:39:53.367424+0000","last_peered":"2023-03-25T04:39:53.367424+0000","last_clean":"2023-03-25T04:39:53.367424+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:39:53.367424+0000","last_undegraded":"2023-03-25T04:39:53.367424+0000","last_fullsized":"2023-03-25T04:39:53.367424+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117185,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:43:00 2023","interfaces":[{"interface":"back","average":{"1min":0.27400000000000002,"5min":0.26700000000000002,"15min":0.26600000000000001},"min":{"1min":0.19600000000000001,"5min":0.17499999999999999,"15min":0.16},"max":{"1min":0.42999999999999999,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.27800000000000002},{"interface":"front","average":{"1min":0.28199999999999997,"5min":0.27500000000000002,"15min":0.27300000000000002},"min":{"1min":0.185,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.47199999999999998,"5min":0.47199999999999998,"15min":0.47199999999999998},"last":0.247}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051776,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:42:54 2023","interfaces":[{"interface":"back","average":{"1min":0.26300000000000001,"5min":0.27000000000000002,"15min":0.26900000000000002},"min":{"1min":0.17399999999999999,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.438,"5min":0.44,"15min":0.44},"last":0.20100000000000001},{"interface":"front","average":{"1min":0.27400000000000002,"5min":0.28199999999999997,"15min":0.28299999999999997},"min":{"1min":0.17599999999999999,"5min":0.17599999999999999,"15min":0.16600000000000001},"max":{"1min":0.41599999999999998,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.23499999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:43:25.042+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:43:25.411+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:43:26.013+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:43:47.937+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:43:48.534+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:43:48.889+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:44:09.318 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:44:09.331 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":586,"stamp":"2023-03-25T04:44:08.288933+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002697"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":40,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:40:23.370056+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:40:23.370056+0000","last_peered":"2023-03-25T04:40:23.370056+0000","last_clean":"2023-03-25T04:40:23.370056+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:40:23.370056+0000","last_undegraded":"2023-03-25T04:40:23.370056+0000","last_fullsized":"2023-03-25T04:40:23.370056+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:40:23.370039+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:40:23.370039+0000","last_peered":"2023-03-25T04:40:23.370039+0000","last_clean":"2023-03-25T04:40:23.370039+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:40:23.370039+0000","last_undegraded":"2023-03-25T04:40:23.370039+0000","last_fullsized":"2023-03-25T04:40:23.370039+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:23.375106+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:41:23.375106+0000","last_peered":"2023-03-25T04:41:23.375106+0000","last_clean":"2023-03-25T04:41:23.375106+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:41:23.375106+0000","last_undegraded":"2023-03-25T04:41:23.375106+0000","last_fullsized":"2023-03-25T04:41:23.375106+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:50.618930+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:41:50.618930+0000","last_peered":"2023-03-25T04:41:50.618930+0000","last_clean":"2023-03-25T04:41:50.618930+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:41:50.618930+0000","last_undegraded":"2023-03-25T04:41:50.618930+0000","last_fullsized":"2023-03-25T04:41:50.618930+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":41,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:58.377821+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:41:58.377821+0000","last_peered":"2023-03-25T04:41:58.377821+0000","last_clean":"2023-03-25T04:41:58.377821+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:41:58.377821+0000","last_undegraded":"2023-03-25T04:41:58.377821+0000","last_fullsized":"2023-03-25T04:41:58.377821+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:58.377836+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:41:58.377836+0000","last_peered":"2023-03-25T04:41:58.377836+0000","last_clean":"2023-03-25T04:41:58.377836+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:41:58.377836+0000","last_undegraded":"2023-03-25T04:41:58.377836+0000","last_fullsized":"2023-03-25T04:41:58.377836+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117197,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:44:01 2023","interfaces":[{"interface":"back","average":{"1min":0.27200000000000002,"5min":0.27000000000000002,"15min":0.26400000000000001},"min":{"1min":0.20399999999999999,"5min":0.17499999999999999,"15min":0.16},"max":{"1min":0.38400000000000001,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.245},{"interface":"front","average":{"1min":0.27600000000000002,"5min":0.27800000000000002,"15min":0.27100000000000002},"min":{"1min":0.23100000000000001,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.40400000000000003,"5min":0.47199999999999998,"15min":0.47199999999999998},"last":0.27500000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051788,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:43:55 2023","interfaces":[{"interface":"back","average":{"1min":0.26100000000000001,"5min":0.26700000000000002,"15min":0.27000000000000002},"min":{"1min":0.17799999999999999,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.374,"5min":0.44,"15min":0.44},"last":0.30299999999999999},{"interface":"front","average":{"1min":0.28299999999999997,"5min":0.28100000000000003,"15min":0.28399999999999997},"min":{"1min":0.20899999999999999,"5min":0.17599999999999999,"15min":0.16600000000000001},"max":{"1min":0.40100000000000002,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.27800000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:44:28.923 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":1126,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":19,"modified":"2023-03-25T04:43:20.278959+0000","services":{}},"progress_events":{}}
, 2023-03-25T04:44:28.936 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now
, 2023-03-25T04:44:28.937 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace: 2023-03-25T04:44:48.786 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:44:48.800 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":606,"stamp":"2023-03-25T04:44:48.296805+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002182"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":41,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:42:34.382196+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:42:34.382196+0000","last_peered":"2023-03-25T04:42:34.382196+0000","last_clean":"2023-03-25T04:42:34.382196+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:42:34.382196+0000","last_undegraded":"2023-03-25T04:42:34.382196+0000","last_fullsized":"2023-03-25T04:42:34.382196+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:42:29.381744+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:42:29.381744+0000","last_peered":"2023-03-25T04:42:29.381744+0000","last_clean":"2023-03-25T04:42:29.381744+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:42:29.381744+0000","last_undegraded":"2023-03-25T04:42:29.381744+0000","last_fullsized":"2023-03-25T04:42:29.381744+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:23.375106+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:41:23.375106+0000","last_peered":"2023-03-25T04:41:23.375106+0000","last_clean":"2023-03-25T04:41:23.375106+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:41:23.375106+0000","last_undegraded":"2023-03-25T04:41:23.375106+0000","last_fullsized":"2023-03-25T04:41:23.375106+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:50.618930+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:41:50.618930+0000","last_peered":"2023-03-25T04:41:50.618930+0000","last_clean":"2023-03-25T04:41:50.618930+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:41:50.618930+0000","last_undegraded":"2023-03-25T04:41:50.618930+0000","last_fullsized":"2023-03-25T04:41:50.618930+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":41,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:58.377821+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:41:58.377821+0000","last_peered":"2023-03-25T04:41:58.377821+0000","last_clean":"2023-03-25T04:41:58.377821+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:41:58.377821+0000","last_undegraded":"2023-03-25T04:41:58.377821+0000","last_fullsized":"2023-03-25T04:41:58.377821+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:58.377836+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:41:58.377836+0000","last_peered":"2023-03-25T04:41:58.377836+0000","last_clean":"2023-03-25T04:41:58.377836+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:41:58.377836+0000","last_undegraded":"2023-03-25T04:41:58.377836+0000","last_fullsized":"2023-03-25T04:41:58.377836+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117206,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:44:01 2023","interfaces":[{"interface":"back","average":{"1min":0.27200000000000002,"5min":0.27000000000000002,"15min":0.26400000000000001},"min":{"1min":0.20399999999999999,"5min":0.17499999999999999,"15min":0.16},"max":{"1min":0.38400000000000001,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.216},{"interface":"front","average":{"1min":0.27600000000000002,"5min":0.27800000000000002,"15min":0.27100000000000002},"min":{"1min":0.23100000000000001,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.40400000000000003,"5min":0.47199999999999998,"15min":0.47199999999999998},"last":0.23999999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051796,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:43:55 2023","interfaces":[{"interface":"back","average":{"1min":0.26100000000000001,"5min":0.26700000000000002,"15min":0.27000000000000002},"min":{"1min":0.17799999999999999,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.374,"5min":0.44,"15min":0.44},"last":0.25600000000000001},{"interface":"front","average":{"1min":0.28299999999999997,"5min":0.28100000000000003,"15min":0.28399999999999997},"min":{"1min":0.20899999999999999,"5min":0.17599999999999999,"15min":0.16600000000000001},"max":{"1min":0.40100000000000002,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.28000000000000003}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:45:05.762 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:45:06.073 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all

Ignoring malparsed backtrace: 2023-03-25T04:45:25.658 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":624,"stamp":"2023-03-25T04:45:24.304294+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002366"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":41,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:42:34.382196+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:42:34.382196+0000","last_peered":"2023-03-25T04:42:34.382196+0000","last_clean":"2023-03-25T04:42:34.382196+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:42:34.382196+0000","last_undegraded":"2023-03-25T04:42:34.382196+0000","last_fullsized":"2023-03-25T04:42:34.382196+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:42:29.381744+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:42:29.381744+0000","last_peered":"2023-03-25T04:42:29.381744+0000","last_clean":"2023-03-25T04:42:29.381744+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:42:29.381744+0000","last_undegraded":"2023-03-25T04:42:29.381744+0000","last_fullsized":"2023-03-25T04:42:29.381744+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:23.375106+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:41:23.375106+0000","last_peered":"2023-03-25T04:41:23.375106+0000","last_clean":"2023-03-25T04:41:23.375106+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:41:23.375106+0000","last_undegraded":"2023-03-25T04:41:23.375106+0000","last_fullsized":"2023-03-25T04:41:23.375106+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:50.618930+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:41:50.618930+0000","last_peered":"2023-03-25T04:41:50.618930+0000","last_clean":"2023-03-25T04:41:50.618930+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:41:50.618930+0000","last_undegraded":"2023-03-25T04:41:50.618930+0000","last_fullsized":"2023-03-25T04:41:50.618930+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":41,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:58.377821+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:41:58.377821+0000","last_peered":"2023-03-25T04:41:58.377821+0000","last_clean":"2023-03-25T04:41:58.377821+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:41:58.377821+0000","last_undegraded":"2023-03-25T04:41:58.377821+0000","last_fullsized":"2023-03-25T04:41:58.377821+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:58.377836+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:41:58.377836+0000","last_peered":"2023-03-25T04:41:58.377836+0000","last_clean":"2023-03-25T04:41:58.377836+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:41:58.377836+0000","last_undegraded":"2023-03-25T04:41:58.377836+0000","last_fullsized":"2023-03-25T04:41:58.377836+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117213,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:45:01 2023","interfaces":[{"interface":"back","average":{"1min":0.251,"5min":0.26800000000000002,"15min":0.26400000000000001},"min":{"1min":0.183,"5min":0.17499999999999999,"15min":0.16},"max":{"1min":0.41499999999999998,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.26900000000000002},{"interface":"front","average":{"1min":0.28000000000000003,"5min":0.27900000000000003,"15min":0.27100000000000002},"min":{"1min":0.20999999999999999,"5min":0.185,"15min":0.17499999999999999},"max":{"1min":0.44400000000000001,"5min":0.47199999999999998,"15min":0.47199999999999998},"last":0.27900000000000003}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051803,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:44:55 2023","interfaces":[{"interface":"back","average":{"1min":0.252,"5min":0.26500000000000001,"15min":0.26900000000000002},"min":{"1min":0.183,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.39700000000000002,"5min":0.44,"15min":0.44},"last":0.20699999999999999},{"interface":"front","average":{"1min":0.27000000000000002,"5min":0.28100000000000003,"15min":0.28299999999999997},"min":{"1min":0.20300000000000001,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":0.42099999999999999,"5min":0.47599999999999998,"15min":0.48599999999999999},"last":0.23599999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:45:25.659 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:45:25.969 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:45:42.735+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:45:42.744+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:45:42.897 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace: 2023-03-25T04:46:00.183 INFO:teuthology.orchestra.run.smithi070.stdout:
, 2023-03-25T04:46:00.183 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":1217,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":20,"modified":"2023-03-25T04:45:42.307700+0000","services":{"mgr":{"daemons":{"summary":"","x":{"start_epoch":0,"start_stamp":"0.000000","gid":0,"addr":"(unrecognized address family 0)/0","metadata":{},"task_status":{}}}}}},"progress_events":{}}
, 2023-03-25T04:46:00.197 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now

Ignoring malparsed backtrace: 2023-03-25T04:46:19.910+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:46:20.039 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all

Ignoring malparsed backtrace: 2023-03-25T04:46:39.611 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}
, 2023-03-25T04:46:39.611 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json
, 2023-03-25T04:46:39.613+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:46:58.485+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:46:58.764+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:46:59.519+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:47:14.132 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:47:14.146 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":678,"stamp":"2023-03-25T04:47:12.325044+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002571"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":42,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:44.393426+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:44:44.393426+0000","last_peered":"2023-03-25T04:44:44.393426+0000","last_clean":"2023-03-25T04:44:44.393426+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:44:44.393426+0000","last_undegraded":"2023-03-25T04:44:44.393426+0000","last_fullsized":"2023-03-25T04:44:44.393426+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:39.393044+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:44:39.393044+0000","last_peered":"2023-03-25T04:44:39.393044+0000","last_clean":"2023-03-25T04:44:39.393044+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:44:39.393044+0000","last_undegraded":"2023-03-25T04:44:39.393044+0000","last_fullsized":"2023-03-25T04:44:39.393044+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:43:24.386177+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:43:24.386177+0000","last_peered":"2023-03-25T04:43:24.386177+0000","last_clean":"2023-03-25T04:43:24.386177+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:43:24.386177+0000","last_undegraded":"2023-03-25T04:43:24.386177+0000","last_fullsized":"2023-03-25T04:43:24.386177+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:43:55.629798+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:43:55.629798+0000","last_peered":"2023-03-25T04:43:55.629798+0000","last_clean":"2023-03-25T04:43:55.629798+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:43:55.629798+0000","last_undegraded":"2023-03-25T04:43:55.629798+0000","last_fullsized":"2023-03-25T04:43:55.629798+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":42,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:04.389904+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:44:04.389904+0000","last_peered":"2023-03-25T04:44:04.389904+0000","last_clean":"2023-03-25T04:44:04.389904+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:44:04.389904+0000","last_undegraded":"2023-03-25T04:44:04.389904+0000","last_fullsized":"2023-03-25T04:44:04.389904+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:04.389923+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:44:04.389923+0000","last_peered":"2023-03-25T04:44:04.389923+0000","last_clean":"2023-03-25T04:44:04.389923+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:44:04.389923+0000","last_undegraded":"2023-03-25T04:44:04.389923+0000","last_fullsized":"2023-03-25T04:44:04.389923+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117235,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:47:05 2023","interfaces":[{"interface":"back","average":{"1min":0.27800000000000002,"5min":0.27000000000000002,"15min":0.26300000000000001},"min":{"1min":0.20100000000000001,"5min":0.16900000000000001,"15min":0.16},"max":{"1min":0.39200000000000002,"5min":0.42999999999999999,"15min":0.45600000000000002},"last":0.379},{"interface":"front","average":{"1min":0.28399999999999997,"5min":0.28100000000000003,"15min":0.27200000000000002},"min":{"1min":0.17799999999999999,"5min":0.17799999999999999,"15min":0.17699999999999999},"max":{"1min":0.42599999999999999,"5min":0.47199999999999998,"15min":0.47199999999999998},"last":0.39800000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051825,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:47:01 2023","interfaces":[{"interface":"back","average":{"1min":0.27300000000000002,"5min":0.26100000000000001,"15min":0.27000000000000002},"min":{"1min":0.186,"5min":0.17399999999999999,"15min":0.17000000000000001},"max":{"1min":0.40400000000000003,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.23899999999999999},{"interface":"front","average":{"1min":0.28699999999999998,"5min":0.27500000000000002,"15min":0.28299999999999997},"min":{"1min":0.186,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.48599999999999999},"last":0.19600000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:47:31.198 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json
, 2023-03-25T04:47:31.411+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:47:31.789 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:47:49.509 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":696,"stamp":"2023-03-25T04:47:48.332093+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002697"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":42,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:44.393426+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:44:44.393426+0000","last_peered":"2023-03-25T04:44:44.393426+0000","last_clean":"2023-03-25T04:44:44.393426+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:44:44.393426+0000","last_undegraded":"2023-03-25T04:44:44.393426+0000","last_fullsized":"2023-03-25T04:44:44.393426+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:39.393044+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:44:39.393044+0000","last_peered":"2023-03-25T04:44:39.393044+0000","last_clean":"2023-03-25T04:44:39.393044+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:44:39.393044+0000","last_undegraded":"2023-03-25T04:44:39.393044+0000","last_fullsized":"2023-03-25T04:44:39.393044+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:45:24.396717+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:45:24.396717+0000","last_peered":"2023-03-25T04:45:24.396717+0000","last_clean":"2023-03-25T04:45:24.396717+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:45:24.396717+0000","last_undegraded":"2023-03-25T04:45:24.396717+0000","last_fullsized":"2023-03-25T04:45:24.396717+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:43:55.629798+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:43:55.629798+0000","last_peered":"2023-03-25T04:43:55.629798+0000","last_clean":"2023-03-25T04:43:55.629798+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:43:55.629798+0000","last_undegraded":"2023-03-25T04:43:55.629798+0000","last_fullsized":"2023-03-25T04:43:55.629798+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":42,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:04.389904+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:44:04.389904+0000","last_peered":"2023-03-25T04:44:04.389904+0000","last_clean":"2023-03-25T04:44:04.389904+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:44:04.389904+0000","last_undegraded":"2023-03-25T04:44:04.389904+0000","last_fullsized":"2023-03-25T04:44:04.389904+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:04.389923+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:44:04.389923+0000","last_peered":"2023-03-25T04:44:04.389923+0000","last_clean":"2023-03-25T04:44:04.389923+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:44:04.389923+0000","last_undegraded":"2023-03-25T04:44:04.389923+0000","last_fullsized":"2023-03-25T04:44:04.389923+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117242,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:47:05 2023","interfaces":[{"interface":"back","average":{"1min":0.27800000000000002,"5min":0.27000000000000002,"15min":0.26300000000000001},"min":{"1min":0.20100000000000001,"5min":0.16900000000000001,"15min":0.16},"max":{"1min":0.39200000000000002,"5min":0.42999999999999999,"15min":0.45600000000000002},"last":0.40500000000000003},{"interface":"front","average":{"1min":0.28399999999999997,"5min":0.28100000000000003,"15min":0.27200000000000002},"min":{"1min":0.17799999999999999,"5min":0.17799999999999999,"15min":0.17699999999999999},"max":{"1min":0.42599999999999999,"5min":0.47199999999999998,"15min":0.47199999999999998},"last":0.42399999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051834,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:47:01 2023","interfaces":[{"interface":"back","average":{"1min":0.27300000000000002,"5min":0.26100000000000001,"15min":0.27000000000000002},"min":{"1min":0.186,"5min":0.17399999999999999,"15min":0.17000000000000001},"max":{"1min":0.40400000000000003,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.25600000000000001},{"interface":"front","average":{"1min":0.28699999999999998,"5min":0.27500000000000002,"15min":0.28299999999999997},"min":{"1min":0.186,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.48599999999999999},"last":0.22800000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:47:49.510 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace:     ctx.managers[config['cluster']].wait_for_clean()
,   File "/home/teuthworker/src/github.com_ceph_ceph-c_511015c23eef0684a64171b710e432448df2b7c7/qa/tasks/ceph_manager.py", line 2747, in wait_for_clean
,     assert time.time() - start < timeout, \

Ignoring malparsed backtrace:  ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
,  1: /lib64/libpthread.so.0(+0x12cf0) [0x7f1453c2fcf0]
,  2: __pthread_mutex_lock()

Ignoring malparsed backtrace: 2023-03-25T04:28:05.172 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":142,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":5,"modified":"2023-03-25T04:27:12.092098+0000","services":{}},"progress_events":{}}
, 2023-03-25T04:28:05.185 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now
, 2023-03-25T04:28:05.186 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace: 2023-03-25T04:28:28.360 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":116,"stamp":"2023-03-25T04:28:28.108007+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002578"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.314661+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:27:45.314661+0000","last_peered":"2023-03-25T04:27:45.314661+0000","last_clean":"2023-03-25T04:27:45.314661+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:27:45.314661+0000","last_undegraded":"2023-03-25T04:27:45.314661+0000","last_fullsized":"2023-03-25T04:27:45.314661+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":25,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.310745+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:27:44.310745+0000","last_peered":"2023-03-25T04:27:44.310745+0000","last_clean":"2023-03-25T04:27:44.310745+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:27:44.310745+0000","last_undegraded":"2023-03-25T04:27:44.310745+0000","last_fullsized":"2023-03-25T04:27:44.310745+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":25,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.310650+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:27:44.310650+0000","last_peered":"2023-03-25T04:27:44.310650+0000","last_clean":"2023-03-25T04:27:44.310650+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:27:44.310650+0000","last_undegraded":"2023-03-25T04:27:44.310650+0000","last_fullsized":"2023-03-25T04:27:44.310650+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":26,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.312962+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:27:44.312962+0000","last_peered":"2023-03-25T04:27:44.312962+0000","last_clean":"2023-03-25T04:27:44.312962+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:27:44.312962+0000","last_undegraded":"2023-03-25T04:27:44.312962+0000","last_fullsized":"2023-03-25T04:27:44.312962+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315064+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:27:45.315064+0000","last_peered":"2023-03-25T04:27:45.315064+0000","last_clean":"2023-03-25T04:27:45.315064+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:27:45.315064+0000","last_undegraded":"2023-03-25T04:27:45.315064+0000","last_fullsized":"2023-03-25T04:27:45.315064+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":25,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.310707+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:27:44.310707+0000","last_peered":"2023-03-25T04:27:44.310707+0000","last_clean":"2023-03-25T04:27:44.310707+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:27:44.310707+0000","last_undegraded":"2023-03-25T04:27:44.310707+0000","last_fullsized":"2023-03-25T04:27:44.310707+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117007,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.375}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051597,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.186}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:28:28.362 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean
, 2023-03-25T04:28:28.362 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:28:45.655 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":124,"stamp":"2023-03-25T04:28:44.111265+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002492"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.314661+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:27:45.314661+0000","last_peered":"2023-03-25T04:27:45.314661+0000","last_clean":"2023-03-25T04:27:45.314661+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:27:45.314661+0000","last_undegraded":"2023-03-25T04:27:45.314661+0000","last_fullsized":"2023-03-25T04:27:45.314661+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":25,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.310745+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:27:44.310745+0000","last_peered":"2023-03-25T04:27:44.310745+0000","last_clean":"2023-03-25T04:27:44.310745+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:27:44.310745+0000","last_undegraded":"2023-03-25T04:27:44.310745+0000","last_fullsized":"2023-03-25T04:27:44.310745+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":25,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.310650+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:27:44.310650+0000","last_peered":"2023-03-25T04:27:44.310650+0000","last_clean":"2023-03-25T04:27:44.310650+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:27:44.310650+0000","last_undegraded":"2023-03-25T04:27:44.310650+0000","last_fullsized":"2023-03-25T04:27:44.310650+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":26,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.312962+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:27:44.312962+0000","last_peered":"2023-03-25T04:27:44.312962+0000","last_clean":"2023-03-25T04:27:44.312962+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:27:44.312962+0000","last_undegraded":"2023-03-25T04:27:44.312962+0000","last_fullsized":"2023-03-25T04:27:44.312962+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315064+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:27:45.315064+0000","last_peered":"2023-03-25T04:27:45.315064+0000","last_clean":"2023-03-25T04:27:45.315064+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:27:45.315064+0000","last_undegraded":"2023-03-25T04:27:45.315064+0000","last_fullsized":"2023-03-25T04:27:45.315064+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":25,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.310707+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:27:44.310707+0000","last_peered":"2023-03-25T04:27:44.310707+0000","last_clean":"2023-03-25T04:27:44.310707+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:27:44.310707+0000","last_undegraded":"2023-03-25T04:27:44.310707+0000","last_fullsized":"2023-03-25T04:27:44.310707+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117011,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:28:31 2023","interfaces":[{"interface":"back","average":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.36899999999999999},{"interface":"front","average":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"min":{"1min":0.19,"5min":0.19,"15min":0.19},"max":{"1min":0.46300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.39800000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051601,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:28:24 2023","interfaces":[{"interface":"back","average":{"1min":0.249,"5min":0.249,"15min":0.249},"min":{"1min":0.17699999999999999,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"last":0.254},{"interface":"front","average":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"min":{"1min":0.20200000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"last":0.312}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:28:45.656 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace: 2023-03-25T04:29:05.171 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}
, 2023-03-25T04:29:05.171 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json
, 2023-03-25T04:29:05.561 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:29:22.465 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:29:22.773 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all

Ignoring malparsed backtrace: 2023-03-25T04:29:42.405 INFO:teuthology.orchestra.run.smithi070.stdout:
, 2023-03-25T04:29:42.406 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":240,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":9,"modified":"2023-03-25T04:29:32.120358+0000","services":{"mon":{"daemons":{"summary":"","a":{"start_epoch":0,"start_stamp":"0.000000","gid":0,"addr":"(unrecognized address family 0)/0","metadata":{},"task_status":{}}}}}},"progress_events":{}}
, 2023-03-25T04:29:42.419 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:29:59.627 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":161,"stamp":"2023-03-25T04:29:58.125186+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002398"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315468+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:27:45.315468+0000","last_peered":"2023-03-25T04:27:45.315468+0000","last_clean":"2023-03-25T04:27:45.315468+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:27:45.315468+0000","last_undegraded":"2023-03-25T04:27:45.315468+0000","last_fullsized":"2023-03-25T04:27:45.315468+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312845+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:27:45.312845+0000","last_peered":"2023-03-25T04:27:45.312845+0000","last_clean":"2023-03-25T04:27:45.312845+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:27:45.312845+0000","last_undegraded":"2023-03-25T04:27:45.312845+0000","last_fullsized":"2023-03-25T04:27:45.312845+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312882+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:27:45.312882+0000","last_peered":"2023-03-25T04:27:45.312882+0000","last_clean":"2023-03-25T04:27:45.312882+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:27:45.312882+0000","last_undegraded":"2023-03-25T04:27:45.312882+0000","last_fullsized":"2023-03-25T04:27:45.312882+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.311730+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:27:45.311730+0000","last_peered":"2023-03-25T04:27:45.311730+0000","last_clean":"2023-03-25T04:27:45.311730+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:27:45.311730+0000","last_undegraded":"2023-03-25T04:27:45.311730+0000","last_fullsized":"2023-03-25T04:27:45.311730+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315735+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:27:45.315735+0000","last_peered":"2023-03-25T04:27:45.315735+0000","last_clean":"2023-03-25T04:27:45.315735+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:27:45.315735+0000","last_undegraded":"2023-03-25T04:27:45.315735+0000","last_fullsized":"2023-03-25T04:27:45.315735+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312833+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:27:45.312833+0000","last_peered":"2023-03-25T04:27:45.312833+0000","last_clean":"2023-03-25T04:27:45.312833+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:27:45.312833+0000","last_undegraded":"2023-03-25T04:27:45.312833+0000","last_fullsized":"2023-03-25T04:27:45.312833+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117026,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:29:33 2023","interfaces":[{"interface":"back","average":{"1min":0.26300000000000001,"5min":0.29099999999999998,"15min":0.29599999999999999},"min":{"1min":0.191,"5min":0.191,"15min":0.191},"max":{"1min":0.371,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.20499999999999999},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.29599999999999999,"15min":0.29999999999999999},"min":{"1min":0.21099999999999999,"5min":0.19,"15min":0.19},"max":{"1min":0.39800000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.19500000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051616,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:29:29 2023","interfaces":[{"interface":"back","average":{"1min":0.27200000000000002,"5min":0.253,"15min":0.25},"min":{"1min":0.183,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"last":0.26400000000000001},{"interface":"front","average":{"1min":0.28499999999999998,"5min":0.26800000000000002,"15min":0.26500000000000001},"min":{"1min":0.16600000000000001,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"last":0.29899999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:29:59.629 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean

Ignoring malparsed backtrace: 2023-03-25T04:30:16.889 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:30:16.903 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":170,"stamp":"2023-03-25T04:30:16.128427+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002222"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315468+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:27:45.315468+0000","last_peered":"2023-03-25T04:27:45.315468+0000","last_clean":"2023-03-25T04:27:45.315468+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:27:45.315468+0000","last_undegraded":"2023-03-25T04:27:45.315468+0000","last_fullsized":"2023-03-25T04:27:45.315468+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312845+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:27:45.312845+0000","last_peered":"2023-03-25T04:27:45.312845+0000","last_clean":"2023-03-25T04:27:45.312845+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:27:45.312845+0000","last_undegraded":"2023-03-25T04:27:45.312845+0000","last_fullsized":"2023-03-25T04:27:45.312845+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312882+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:27:45.312882+0000","last_peered":"2023-03-25T04:27:45.312882+0000","last_clean":"2023-03-25T04:27:45.312882+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:27:45.312882+0000","last_undegraded":"2023-03-25T04:27:45.312882+0000","last_fullsized":"2023-03-25T04:27:45.312882+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.311730+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:27:45.311730+0000","last_peered":"2023-03-25T04:27:45.311730+0000","last_clean":"2023-03-25T04:27:45.311730+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:27:45.311730+0000","last_undegraded":"2023-03-25T04:27:45.311730+0000","last_fullsized":"2023-03-25T04:27:45.311730+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315735+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:27:45.315735+0000","last_peered":"2023-03-25T04:27:45.315735+0000","last_clean":"2023-03-25T04:27:45.315735+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:27:45.315735+0000","last_undegraded":"2023-03-25T04:27:45.315735+0000","last_fullsized":"2023-03-25T04:27:45.315735+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312833+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:27:45.312833+0000","last_peered":"2023-03-25T04:27:45.312833+0000","last_clean":"2023-03-25T04:27:45.312833+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:27:45.312833+0000","last_undegraded":"2023-03-25T04:27:45.312833+0000","last_fullsized":"2023-03-25T04:27:45.312833+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117030,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:29:33 2023","interfaces":[{"interface":"back","average":{"1min":0.26300000000000001,"5min":0.29099999999999998,"15min":0.29599999999999999},"min":{"1min":0.191,"5min":0.191,"15min":0.191},"max":{"1min":0.371,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.245},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.29599999999999999,"15min":0.29999999999999999},"min":{"1min":0.21099999999999999,"5min":0.19,"15min":0.19},"max":{"1min":0.39800000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.20899999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051620,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:29:29 2023","interfaces":[{"interface":"back","average":{"1min":0.27200000000000002,"5min":0.253,"15min":0.25},"min":{"1min":0.183,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"last":0.19400000000000001},{"interface":"front","average":{"1min":0.28499999999999998,"5min":0.26800000000000002,"15min":0.26500000000000001},"min":{"1min":0.16600000000000001,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"last":0.23599999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:30:31.984+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:30:32.201+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:30:32.981+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)

Ignoring malparsed backtrace: 2023-03-25T04:30:48.449 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now
, 2023-03-25T04:30:48.450 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:30:48.760 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:31:08.342 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":196,"stamp":"2023-03-25T04:31:08.139018+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002431"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315468+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:27:45.315468+0000","last_peered":"2023-03-25T04:27:45.315468+0000","last_clean":"2023-03-25T04:27:45.315468+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:27:45.315468+0000","last_undegraded":"2023-03-25T04:27:45.315468+0000","last_fullsized":"2023-03-25T04:27:45.315468+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312845+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:27:45.312845+0000","last_peered":"2023-03-25T04:27:45.312845+0000","last_clean":"2023-03-25T04:27:45.312845+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:27:45.312845+0000","last_undegraded":"2023-03-25T04:27:45.312845+0000","last_fullsized":"2023-03-25T04:27:45.312845+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312882+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:27:45.312882+0000","last_peered":"2023-03-25T04:27:45.312882+0000","last_clean":"2023-03-25T04:27:45.312882+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:27:45.312882+0000","last_undegraded":"2023-03-25T04:27:45.312882+0000","last_fullsized":"2023-03-25T04:27:45.312882+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.311730+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:27:45.311730+0000","last_peered":"2023-03-25T04:27:45.311730+0000","last_clean":"2023-03-25T04:27:45.311730+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:27:45.311730+0000","last_undegraded":"2023-03-25T04:27:45.311730+0000","last_fullsized":"2023-03-25T04:27:45.311730+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315735+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:27:45.315735+0000","last_peered":"2023-03-25T04:27:45.315735+0000","last_clean":"2023-03-25T04:27:45.315735+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:27:45.315735+0000","last_undegraded":"2023-03-25T04:27:45.315735+0000","last_fullsized":"2023-03-25T04:27:45.315735+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312833+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:27:45.312833+0000","last_peered":"2023-03-25T04:27:45.312833+0000","last_clean":"2023-03-25T04:27:45.312833+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:27:45.312833+0000","last_undegraded":"2023-03-25T04:27:45.312833+0000","last_fullsized":"2023-03-25T04:27:45.312833+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117040,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:30:37 2023","interfaces":[{"interface":"back","average":{"1min":0.28799999999999998,"5min":0.28899999999999998,"15min":0.29499999999999998},"min":{"1min":0.20200000000000001,"5min":0.191,"15min":0.191},"max":{"1min":0.433,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.29799999999999999},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.28999999999999998,"15min":0.29799999999999999},"min":{"1min":0.17499999999999999,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":0.40300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.25700000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051630,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:30:32 2023","interfaces":[{"interface":"back","average":{"1min":0.25600000000000001,"5min":0.255,"15min":0.251},"min":{"1min":0.19400000000000001,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.35699999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"last":0.22800000000000001},{"interface":"front","average":{"1min":0.28699999999999998,"5min":0.27200000000000002,"15min":0.26600000000000001},"min":{"1min":0.20599999999999999,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.39400000000000002,"5min":0.42899999999999999,"15min":0.42899999999999999},"last":0.24099999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:31:08.343 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace: 2023-03-25T04:31:24.283+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:31:25.192+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:31:25.295+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)

Ignoring malparsed backtrace: 2023-03-25T04:31:40.257 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:31:40.271 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":212,"stamp":"2023-03-25T04:31:40.144732+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002208"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315468+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:27:45.315468+0000","last_peered":"2023-03-25T04:27:45.315468+0000","last_clean":"2023-03-25T04:27:45.315468+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:27:45.315468+0000","last_undegraded":"2023-03-25T04:27:45.315468+0000","last_fullsized":"2023-03-25T04:27:45.315468+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312845+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:27:45.312845+0000","last_peered":"2023-03-25T04:27:45.312845+0000","last_clean":"2023-03-25T04:27:45.312845+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:27:45.312845+0000","last_undegraded":"2023-03-25T04:27:45.312845+0000","last_fullsized":"2023-03-25T04:27:45.312845+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:23.308607+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:29:23.308607+0000","last_peered":"2023-03-25T04:29:23.308607+0000","last_clean":"2023-03-25T04:29:23.308607+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:29:23.308607+0000","last_undegraded":"2023-03-25T04:29:23.308607+0000","last_fullsized":"2023-03-25T04:29:23.308607+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":28,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:20.554710+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:29:20.554710+0000","last_peered":"2023-03-25T04:29:20.554710+0000","last_clean":"2023-03-25T04:29:20.554710+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:29:20.554710+0000","last_undegraded":"2023-03-25T04:29:20.554710+0000","last_fullsized":"2023-03-25T04:29:20.554710+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309216+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:29:28.309216+0000","last_peered":"2023-03-25T04:29:28.309216+0000","last_clean":"2023-03-25T04:29:28.309216+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:29:28.309216+0000","last_undegraded":"2023-03-25T04:29:28.309216+0000","last_fullsized":"2023-03-25T04:29:28.309216+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309233+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:29:28.309233+0000","last_peered":"2023-03-25T04:29:28.309233+0000","last_clean":"2023-03-25T04:29:28.309233+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:29:28.309233+0000","last_undegraded":"2023-03-25T04:29:28.309233+0000","last_fullsized":"2023-03-25T04:29:28.309233+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117046,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:30:37 2023","interfaces":[{"interface":"back","average":{"1min":0.28799999999999998,"5min":0.28899999999999998,"15min":0.29499999999999998},"min":{"1min":0.20200000000000001,"5min":0.191,"15min":0.191},"max":{"1min":0.433,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.216},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.28999999999999998,"15min":0.29799999999999999},"min":{"1min":0.17499999999999999,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":0.40300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.24299999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051637,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:31:35 2023","interfaces":[{"interface":"back","average":{"1min":0.26100000000000001,"5min":0.25700000000000001,"15min":0.251},"min":{"1min":0.19,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.35899999999999999,"5min":0.41599999999999998,"15min":0.41599999999999998},"last":0.221},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.27400000000000002,"15min":0.26700000000000002},"min":{"1min":0.20200000000000001,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.374,"5min":0.42899999999999999,"15min":0.42899999999999999},"last":0.24099999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:32:01.906+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:32:02.388+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:32:02.794 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:32:22.326 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":233,"stamp":"2023-03-25T04:32:22.153289+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002545"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:33.309748+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:29:33.309748+0000","last_peered":"2023-03-25T04:29:33.309748+0000","last_clean":"2023-03-25T04:29:33.309748+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:29:33.309748+0000","last_undegraded":"2023-03-25T04:29:33.309748+0000","last_fullsized":"2023-03-25T04:29:33.309748+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:33.309729+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:29:33.309729+0000","last_peered":"2023-03-25T04:29:33.309729+0000","last_clean":"2023-03-25T04:29:33.309729+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:29:33.309729+0000","last_undegraded":"2023-03-25T04:29:33.309729+0000","last_fullsized":"2023-03-25T04:29:33.309729+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:23.308607+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:29:23.308607+0000","last_peered":"2023-03-25T04:29:23.308607+0000","last_clean":"2023-03-25T04:29:23.308607+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:29:23.308607+0000","last_undegraded":"2023-03-25T04:29:23.308607+0000","last_fullsized":"2023-03-25T04:29:23.308607+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":28,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:20.554710+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:29:20.554710+0000","last_peered":"2023-03-25T04:29:20.554710+0000","last_clean":"2023-03-25T04:29:20.554710+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:29:20.554710+0000","last_undegraded":"2023-03-25T04:29:20.554710+0000","last_fullsized":"2023-03-25T04:29:20.554710+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309216+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:29:28.309216+0000","last_peered":"2023-03-25T04:29:28.309216+0000","last_clean":"2023-03-25T04:29:28.309216+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:29:28.309216+0000","last_undegraded":"2023-03-25T04:29:28.309216+0000","last_fullsized":"2023-03-25T04:29:28.309216+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309233+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:29:28.309233+0000","last_peered":"2023-03-25T04:29:28.309233+0000","last_clean":"2023-03-25T04:29:28.309233+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:29:28.309233+0000","last_undegraded":"2023-03-25T04:29:28.309233+0000","last_fullsized":"2023-03-25T04:29:28.309233+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117055,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:31:42 2023","interfaces":[{"interface":"back","average":{"1min":0.27500000000000002,"5min":0.28399999999999997,"15min":0.29399999999999998},"min":{"1min":0.19800000000000001,"5min":0.191,"15min":0.191},"max":{"1min":0.36899999999999999,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.32300000000000001},{"interface":"front","average":{"1min":0.27900000000000003,"5min":0.28499999999999998,"15min":0.29599999999999999},"min":{"1min":0.216,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":0.40799999999999997,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.27100000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051646,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:31:35 2023","interfaces":[{"interface":"back","average":{"1min":0.26100000000000001,"5min":0.25700000000000001,"15min":0.251},"min":{"1min":0.19,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.35899999999999999,"5min":0.41599999999999998,"15min":0.41599999999999998},"last":0.25600000000000001},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.27400000000000002,"15min":0.26700000000000002},"min":{"1min":0.20200000000000001,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.374,"5min":0.42899999999999999,"15min":0.42899999999999999},"last":0.29199999999999998}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:32:22.327 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean
, 2023-03-25T04:32:22.327 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}

Ignoring malparsed backtrace: 2023-03-25T04:32:39.836 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:32:39.849 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":241,"stamp":"2023-03-25T04:32:38.156312+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002346"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:33.309748+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:29:33.309748+0000","last_peered":"2023-03-25T04:29:33.309748+0000","last_clean":"2023-03-25T04:29:33.309748+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:29:33.309748+0000","last_undegraded":"2023-03-25T04:29:33.309748+0000","last_fullsized":"2023-03-25T04:29:33.309748+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:33.309729+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:29:33.309729+0000","last_peered":"2023-03-25T04:29:33.309729+0000","last_clean":"2023-03-25T04:29:33.309729+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:29:33.309729+0000","last_undegraded":"2023-03-25T04:29:33.309729+0000","last_fullsized":"2023-03-25T04:29:33.309729+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:23.308607+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:29:23.308607+0000","last_peered":"2023-03-25T04:29:23.308607+0000","last_clean":"2023-03-25T04:29:23.308607+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:29:23.308607+0000","last_undegraded":"2023-03-25T04:29:23.308607+0000","last_fullsized":"2023-03-25T04:29:23.308607+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":28,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:20.554710+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:29:20.554710+0000","last_peered":"2023-03-25T04:29:20.554710+0000","last_clean":"2023-03-25T04:29:20.554710+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:29:20.554710+0000","last_undegraded":"2023-03-25T04:29:20.554710+0000","last_fullsized":"2023-03-25T04:29:20.554710+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309216+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:29:28.309216+0000","last_peered":"2023-03-25T04:29:28.309216+0000","last_clean":"2023-03-25T04:29:28.309216+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:29:28.309216+0000","last_undegraded":"2023-03-25T04:29:28.309216+0000","last_fullsized":"2023-03-25T04:29:28.309216+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309233+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:29:28.309233+0000","last_peered":"2023-03-25T04:29:28.309233+0000","last_clean":"2023-03-25T04:29:28.309233+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:29:28.309233+0000","last_undegraded":"2023-03-25T04:29:28.309233+0000","last_fullsized":"2023-03-25T04:29:28.309233+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117058,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:31:42 2023","interfaces":[{"interface":"back","average":{"1min":0.27500000000000002,"5min":0.28399999999999997,"15min":0.29399999999999998},"min":{"1min":0.19800000000000001,"5min":0.191,"15min":0.191},"max":{"1min":0.36899999999999999,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.30599999999999999},{"interface":"front","average":{"1min":0.27900000000000003,"5min":0.28499999999999998,"15min":0.29599999999999999},"min":{"1min":0.216,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":0.40799999999999997,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.32500000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051649,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:31:35 2023","interfaces":[{"interface":"back","average":{"1min":0.26100000000000001,"5min":0.25700000000000001,"15min":0.251},"min":{"1min":0.19,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.35899999999999999,"5min":0.41599999999999998,"15min":0.41599999999999998},"last":0.24199999999999999},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.27400000000000002,"15min":0.26700000000000002},"min":{"1min":0.20200000000000001,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.374,"5min":0.42899999999999999,"15min":0.42899999999999999},"last":0.22600000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:32:54.944+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:32:55.469+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:32:55.927+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)

Ignoring malparsed backtrace: 2023-03-25T04:33:13.958 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean
, 2023-03-25T04:33:13.959 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}
, 2023-03-25T04:33:13.959 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json

Ignoring malparsed backtrace: 2023-03-25T04:33:31.199 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":267,"stamp":"2023-03-25T04:33:30.166660+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002269"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:33.309748+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:29:33.309748+0000","last_peered":"2023-03-25T04:29:33.309748+0000","last_clean":"2023-03-25T04:29:33.309748+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:29:33.309748+0000","last_undegraded":"2023-03-25T04:29:33.309748+0000","last_fullsized":"2023-03-25T04:29:33.309748+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:33.309729+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:29:33.309729+0000","last_peered":"2023-03-25T04:29:33.309729+0000","last_clean":"2023-03-25T04:29:33.309729+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:29:33.309729+0000","last_undegraded":"2023-03-25T04:29:33.309729+0000","last_fullsized":"2023-03-25T04:29:33.309729+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":28,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:31:23.320894+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:31:23.320894+0000","last_peered":"2023-03-25T04:31:23.320894+0000","last_clean":"2023-03-25T04:31:23.320894+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:31:23.320894+0000","last_undegraded":"2023-03-25T04:31:23.320894+0000","last_fullsized":"2023-03-25T04:31:23.320894+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":28,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:20.554710+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:29:20.554710+0000","last_peered":"2023-03-25T04:29:20.554710+0000","last_clean":"2023-03-25T04:29:20.554710+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:29:20.554710+0000","last_undegraded":"2023-03-25T04:29:20.554710+0000","last_fullsized":"2023-03-25T04:29:20.554710+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309216+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:29:28.309216+0000","last_peered":"2023-03-25T04:29:28.309216+0000","last_clean":"2023-03-25T04:29:28.309216+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:29:28.309216+0000","last_undegraded":"2023-03-25T04:29:28.309216+0000","last_fullsized":"2023-03-25T04:29:28.309216+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309233+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:29:28.309233+0000","last_peered":"2023-03-25T04:29:28.309233+0000","last_clean":"2023-03-25T04:29:28.309233+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:29:28.309233+0000","last_undegraded":"2023-03-25T04:29:28.309233+0000","last_fullsized":"2023-03-25T04:29:28.309233+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117068,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:32:46 2023","interfaces":[{"interface":"back","average":{"1min":0.27000000000000002,"5min":0.27900000000000003,"15min":0.29199999999999998},"min":{"1min":0.17899999999999999,"5min":0.17899999999999999,"15min":0.17899999999999999},"max":{"1min":0.35599999999999998,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.28799999999999998},{"interface":"front","average":{"1min":0.26800000000000002,"5min":0.27800000000000002,"15min":0.29399999999999998},"min":{"1min":0.19900000000000001,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":0.377,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.315}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051660,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:32:36 2023","interfaces":[{"interface":"back","average":{"1min":0.248,"5min":0.25700000000000001,"15min":0.251},"min":{"1min":0.17000000000000001,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"last":0.22700000000000001},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.27600000000000002,"15min":0.26800000000000002},"min":{"1min":0.192,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"last":0.20399999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:33:31.201 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:33:31.400+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)

Ignoring malparsed backtrace: 2023-03-25T04:33:47.531+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:33:47.951+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:33:48.505+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)

Ignoring malparsed backtrace: 2023-03-25T04:34:06.104 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json
, 2023-03-25T04:34:06.118+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:34:06.497 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:34:23.399 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:34:23.665+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:34:23.712 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:34:40.195+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:34:40.523+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:34:40.676 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace: 2023-03-25T04:35:03.381+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:35:04.139+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:35:04.391+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)

Ignoring malparsed backtrace: 2023-03-25T04:35:22.738 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean
, 2023-03-25T04:35:22.739 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}
, 2023-03-25T04:35:22.739 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json

Ignoring malparsed backtrace: 2023-03-25T04:35:41.393+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:35:42.352+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:35:42.376+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:35:57.384 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":340,"stamp":"2023-03-25T04:35:56.194992+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.001930"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":36,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:31:43.322894+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:31:43.322894+0000","last_peered":"2023-03-25T04:31:43.322894+0000","last_clean":"2023-03-25T04:31:43.322894+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:31:43.322894+0000","last_undegraded":"2023-03-25T04:31:43.322894+0000","last_fullsized":"2023-03-25T04:31:43.322894+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":28,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:31:43.322876+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:31:43.322876+0000","last_peered":"2023-03-25T04:31:43.322876+0000","last_clean":"2023-03-25T04:31:43.322876+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:31:43.322876+0000","last_undegraded":"2023-03-25T04:31:43.322876+0000","last_fullsized":"2023-03-25T04:31:43.322876+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:23.331998+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:33:23.331998+0000","last_peered":"2023-03-25T04:33:23.331998+0000","last_clean":"2023-03-25T04:33:23.331998+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:33:23.331998+0000","last_undegraded":"2023-03-25T04:33:23.331998+0000","last_fullsized":"2023-03-25T04:33:23.331998+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:30.576723+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:33:30.576723+0000","last_peered":"2023-03-25T04:33:30.576723+0000","last_clean":"2023-03-25T04:33:30.576723+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:33:30.576723+0000","last_undegraded":"2023-03-25T04:33:30.576723+0000","last_fullsized":"2023-03-25T04:33:30.576723+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":37,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:38.333447+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:33:38.333447+0000","last_peered":"2023-03-25T04:33:38.333447+0000","last_clean":"2023-03-25T04:33:38.333447+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:33:38.333447+0000","last_undegraded":"2023-03-25T04:33:38.333447+0000","last_fullsized":"2023-03-25T04:33:38.333447+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:38.333467+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:33:38.333467+0000","last_peered":"2023-03-25T04:33:38.333467+0000","last_clean":"2023-03-25T04:33:38.333467+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:33:38.333467+0000","last_undegraded":"2023-03-25T04:33:38.333467+0000","last_fullsized":"2023-03-25T04:33:38.333467+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117098,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:35:49 2023","interfaces":[{"interface":"back","average":{"1min":0.255,"5min":0.26000000000000001,"15min":0.28199999999999997},"min":{"1min":0.17899999999999999,"5min":0.16,"15min":0.16},"max":{"1min":0.36399999999999999,"5min":0.42599999999999999,"15min":0.45100000000000001},"last":0.26100000000000001},{"interface":"front","average":{"1min":0.26000000000000001,"5min":0.26700000000000002,"15min":0.28599999999999998},"min":{"1min":0.188,"5min":0.17999999999999999,"15min":0.17499999999999999},"max":{"1min":0.39100000000000001,"5min":0.45200000000000001,"15min":0.46300000000000002},"last":0.28399999999999997}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051689,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:35:42 2023","interfaces":[{"interface":"back","average":{"1min":0.29399999999999998,"5min":0.27200000000000002,"15min":0.25800000000000001},"min":{"1min":0.17299999999999999,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.41099999999999998,"5min":0.42699999999999999,"15min":0.42699999999999999},"last":0.20399999999999999},{"interface":"front","average":{"1min":0.30299999999999999,"5min":0.28399999999999997,"15min":0.27300000000000002},"min":{"1min":0.19700000000000001,"5min":0.192,"15min":0.16600000000000001},"max":{"1min":0.43099999999999999,"5min":0.45800000000000002,"15min":0.45800000000000002},"last":0.22}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:35:57.624+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)

Ignoring malparsed backtrace: 2023-03-25T04:36:17.226 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:36:17.239 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":350,"stamp":"2023-03-25T04:36:16.198866+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002148"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":37,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:53.335026+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:33:53.335026+0000","last_peered":"2023-03-25T04:33:53.335026+0000","last_clean":"2023-03-25T04:33:53.335026+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:33:53.335026+0000","last_undegraded":"2023-03-25T04:33:53.335026+0000","last_fullsized":"2023-03-25T04:33:53.335026+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:53.335009+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:33:53.335009+0000","last_peered":"2023-03-25T04:33:53.335009+0000","last_clean":"2023-03-25T04:33:53.335009+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:33:53.335009+0000","last_undegraded":"2023-03-25T04:33:53.335009+0000","last_fullsized":"2023-03-25T04:33:53.335009+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:23.331998+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:33:23.331998+0000","last_peered":"2023-03-25T04:33:23.331998+0000","last_clean":"2023-03-25T04:33:23.331998+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:33:23.331998+0000","last_undegraded":"2023-03-25T04:33:23.331998+0000","last_fullsized":"2023-03-25T04:33:23.331998+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:30.576723+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:33:30.576723+0000","last_peered":"2023-03-25T04:33:30.576723+0000","last_clean":"2023-03-25T04:33:30.576723+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:33:30.576723+0000","last_undegraded":"2023-03-25T04:33:30.576723+0000","last_fullsized":"2023-03-25T04:33:30.576723+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":37,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:38.333447+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:33:38.333447+0000","last_peered":"2023-03-25T04:33:38.333447+0000","last_clean":"2023-03-25T04:33:38.333447+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:33:38.333447+0000","last_undegraded":"2023-03-25T04:33:38.333447+0000","last_fullsized":"2023-03-25T04:33:38.333447+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:38.333467+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:33:38.333467+0000","last_peered":"2023-03-25T04:33:38.333467+0000","last_clean":"2023-03-25T04:33:38.333467+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:33:38.333467+0000","last_undegraded":"2023-03-25T04:33:38.333467+0000","last_fullsized":"2023-03-25T04:33:38.333467+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117102,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:35:49 2023","interfaces":[{"interface":"back","average":{"1min":0.255,"5min":0.26000000000000001,"15min":0.28199999999999997},"min":{"1min":0.17899999999999999,"5min":0.16,"15min":0.16},"max":{"1min":0.36399999999999999,"5min":0.42599999999999999,"15min":0.45100000000000001},"last":0.28399999999999997},{"interface":"front","average":{"1min":0.26000000000000001,"5min":0.26700000000000002,"15min":0.28599999999999998},"min":{"1min":0.188,"5min":0.17999999999999999,"15min":0.17499999999999999},"max":{"1min":0.39100000000000001,"5min":0.45200000000000001,"15min":0.46300000000000002},"last":0.30299999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051693,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:35:42 2023","interfaces":[{"interface":"back","average":{"1min":0.29399999999999998,"5min":0.27200000000000002,"15min":0.25800000000000001},"min":{"1min":0.17299999999999999,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.41099999999999998,"5min":0.42699999999999999,"15min":0.42699999999999999},"last":0.35099999999999998},{"interface":"front","average":{"1min":0.30299999999999999,"5min":0.28399999999999997,"15min":0.27300000000000002},"min":{"1min":0.19700000000000001,"5min":0.192,"15min":0.16600000000000001},"max":{"1min":0.43099999999999999,"5min":0.45800000000000002,"15min":0.45800000000000002},"last":0.32900000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:36:34.556+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:36:34.717+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:36:35.583+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)

Ignoring malparsed backtrace: 2023-03-25T04:36:52.680 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json
, 2023-03-25T04:36:52.701+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:36:53.067 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:37:09.963 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:37:10.273 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all

Ignoring malparsed backtrace: 2023-03-25T04:37:26.856+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:37:27.288 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:37:27.538+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:37:41.903 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":392,"stamp":"2023-03-25T04:37:40.213373+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002012"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":37,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:53.335026+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:33:53.335026+0000","last_peered":"2023-03-25T04:33:53.335026+0000","last_clean":"2023-03-25T04:33:53.335026+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:33:53.335026+0000","last_undegraded":"2023-03-25T04:33:53.335026+0000","last_fullsized":"2023-03-25T04:33:53.335026+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:53.335009+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:33:53.335009+0000","last_peered":"2023-03-25T04:33:53.335009+0000","last_clean":"2023-03-25T04:33:53.335009+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:33:53.335009+0000","last_undegraded":"2023-03-25T04:33:53.335009+0000","last_fullsized":"2023-03-25T04:33:53.335009+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:23.343585+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:35:23.343585+0000","last_peered":"2023-03-25T04:35:23.343585+0000","last_clean":"2023-03-25T04:35:23.343585+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:35:23.343585+0000","last_undegraded":"2023-03-25T04:35:23.343585+0000","last_fullsized":"2023-03-25T04:35:23.343585+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:30.576723+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:33:30.576723+0000","last_peered":"2023-03-25T04:33:30.576723+0000","last_clean":"2023-03-25T04:33:30.576723+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:33:30.576723+0000","last_undegraded":"2023-03-25T04:33:30.576723+0000","last_fullsized":"2023-03-25T04:33:30.576723+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":37,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:38.333447+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:33:38.333447+0000","last_peered":"2023-03-25T04:33:38.333447+0000","last_clean":"2023-03-25T04:33:38.333447+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:33:38.333447+0000","last_undegraded":"2023-03-25T04:33:38.333447+0000","last_fullsized":"2023-03-25T04:33:38.333447+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:38.333467+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:33:38.333467+0000","last_peered":"2023-03-25T04:33:38.333467+0000","last_clean":"2023-03-25T04:33:38.333467+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:33:38.333467+0000","last_undegraded":"2023-03-25T04:33:38.333467+0000","last_fullsized":"2023-03-25T04:33:38.333467+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117119,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:36:50 2023","interfaces":[{"interface":"back","average":{"1min":0.254,"5min":0.25600000000000001,"15min":0.27900000000000003},"min":{"1min":0.20499999999999999,"5min":0.16,"15min":0.16},"max":{"1min":0.371,"5min":0.42599999999999999,"15min":0.45100000000000001},"last":0.23000000000000001},{"interface":"front","average":{"1min":0.26800000000000002,"5min":0.26500000000000001,"15min":0.28399999999999997},"min":{"1min":0.19700000000000001,"5min":0.17999999999999999,"15min":0.17499999999999999},"max":{"1min":0.34699999999999998,"5min":0.45200000000000001,"15min":0.46300000000000002},"last":0.20899999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051710,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:36:44 2023","interfaces":[{"interface":"back","average":{"1min":0.28399999999999997,"5min":0.27700000000000002,"15min":0.26100000000000001},"min":{"1min":0.192,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.40899999999999997,"5min":0.42699999999999999,"15min":0.42699999999999999},"last":0.39700000000000002},{"interface":"front","average":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.27500000000000002},"min":{"1min":0.182,"5min":0.182,"15min":0.16600000000000001},"max":{"1min":0.439,"5min":0.45800000000000002,"15min":0.45800000000000002},"last":0.439}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:37:42.400+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)

Ignoring malparsed backtrace: 2023-03-25T04:38:04.766+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:38:04.922 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:38:05.246 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:38:19.605 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":411,"stamp":"2023-03-25T04:38:18.220966+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002368"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":38,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:36:03.347070+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:36:03.347070+0000","last_peered":"2023-03-25T04:36:03.347070+0000","last_clean":"2023-03-25T04:36:03.347070+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:36:03.347070+0000","last_undegraded":"2023-03-25T04:36:03.347070+0000","last_fullsized":"2023-03-25T04:36:03.347070+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:36:03.347025+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:36:03.347025+0000","last_peered":"2023-03-25T04:36:03.347025+0000","last_clean":"2023-03-25T04:36:03.347025+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:36:03.347025+0000","last_undegraded":"2023-03-25T04:36:03.347025+0000","last_fullsized":"2023-03-25T04:36:03.347025+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:23.343585+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:35:23.343585+0000","last_peered":"2023-03-25T04:35:23.343585+0000","last_clean":"2023-03-25T04:35:23.343585+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:35:23.343585+0000","last_undegraded":"2023-03-25T04:35:23.343585+0000","last_fullsized":"2023-03-25T04:35:23.343585+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:35.587025+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:35:35.587025+0000","last_peered":"2023-03-25T04:35:35.587025+0000","last_clean":"2023-03-25T04:35:35.587025+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:35:35.587025+0000","last_undegraded":"2023-03-25T04:35:35.587025+0000","last_fullsized":"2023-03-25T04:35:35.587025+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":38,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:43.345308+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:35:43.345308+0000","last_peered":"2023-03-25T04:35:43.345308+0000","last_clean":"2023-03-25T04:35:43.345308+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:35:43.345308+0000","last_undegraded":"2023-03-25T04:35:43.345308+0000","last_fullsized":"2023-03-25T04:35:43.345308+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:43.345324+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:35:43.345324+0000","last_peered":"2023-03-25T04:35:43.345324+0000","last_clean":"2023-03-25T04:35:43.345324+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:35:43.345324+0000","last_undegraded":"2023-03-25T04:35:43.345324+0000","last_fullsized":"2023-03-25T04:35:43.345324+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117127,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:37:53 2023","interfaces":[{"interface":"back","average":{"1min":0.25800000000000001,"5min":0.253,"15min":0.27700000000000002},"min":{"1min":0.184,"5min":0.16,"15min":0.16},"max":{"1min":0.44600000000000001,"5min":0.44600000000000001,"15min":0.45100000000000001},"last":0.35199999999999998},{"interface":"front","average":{"1min":0.26900000000000002,"5min":0.26500000000000001,"15min":0.28199999999999997},"min":{"1min":0.192,"5min":0.17999999999999999,"15min":0.17499999999999999},"max":{"1min":0.46400000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"last":0.33200000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051717,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:37:44 2023","interfaces":[{"interface":"back","average":{"1min":0.27300000000000002,"5min":0.28199999999999997,"15min":0.26200000000000001},"min":{"1min":0.17899999999999999,"5min":0.17299999999999999,"15min":0.17000000000000001},"max":{"1min":0.41799999999999998,"5min":0.42099999999999999,"15min":0.42699999999999999},"last":0.28699999999999998},{"interface":"front","average":{"1min":0.29399999999999998,"5min":0.29099999999999998,"15min":0.27700000000000002},"min":{"1min":0.19900000000000001,"5min":0.182,"15min":0.16600000000000001},"max":{"1min":0.441,"5min":0.441,"15min":0.45800000000000002},"last":0.26300000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:38:19.718+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:38:20.315+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)

Ignoring malparsed backtrace: 2023-03-25T04:38:44.283 INFO:teuthology.orchestra.run.smithi070.stdout:
, 2023-03-25T04:38:44.283 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":781,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":16,"modified":"2023-03-25T04:38:40.225321+0000","services":{}},"progress_events":{}}
, 2023-03-25T04:38:44.296 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now

Ignoring malparsed backtrace: 2023-03-25T04:39:02.477+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:39:02.578+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:39:03.524+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:39:20.359 INFO:teuthology.orchestra.run.smithi070.stdout:
, 2023-03-25T04:39:20.359 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":818,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":16,"modified":"2023-03-25T04:38:40.225321+0000","services":{}},"progress_events":{}}
, 2023-03-25T04:39:20.372 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now

Ignoring malparsed backtrace: 2023-03-25T04:39:37.527 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:39:37.541 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":450,"stamp":"2023-03-25T04:39:36.235332+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002086"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":38,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:36:03.347070+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:36:03.347070+0000","last_peered":"2023-03-25T04:36:03.347070+0000","last_clean":"2023-03-25T04:36:03.347070+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:36:03.347070+0000","last_undegraded":"2023-03-25T04:36:03.347070+0000","last_fullsized":"2023-03-25T04:36:03.347070+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:36:03.347025+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:36:03.347025+0000","last_peered":"2023-03-25T04:36:03.347025+0000","last_clean":"2023-03-25T04:36:03.347025+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:36:03.347025+0000","last_undegraded":"2023-03-25T04:36:03.347025+0000","last_fullsized":"2023-03-25T04:36:03.347025+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:23.353778+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:37:23.353778+0000","last_peered":"2023-03-25T04:37:23.353778+0000","last_clean":"2023-03-25T04:37:23.353778+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:37:23.353778+0000","last_undegraded":"2023-03-25T04:37:23.353778+0000","last_fullsized":"2023-03-25T04:37:23.353778+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:35.587025+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:35:35.587025+0000","last_peered":"2023-03-25T04:35:35.587025+0000","last_clean":"2023-03-25T04:35:35.587025+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:35:35.587025+0000","last_undegraded":"2023-03-25T04:35:35.587025+0000","last_fullsized":"2023-03-25T04:35:35.587025+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":38,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:43.345308+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:35:43.345308+0000","last_peered":"2023-03-25T04:35:43.345308+0000","last_clean":"2023-03-25T04:35:43.345308+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:35:43.345308+0000","last_undegraded":"2023-03-25T04:35:43.345308+0000","last_fullsized":"2023-03-25T04:35:43.345308+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:43.345324+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:35:43.345324+0000","last_peered":"2023-03-25T04:35:43.345324+0000","last_clean":"2023-03-25T04:35:43.345324+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:35:43.345324+0000","last_undegraded":"2023-03-25T04:35:43.345324+0000","last_fullsized":"2023-03-25T04:35:43.345324+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117143,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:38:54 2023","interfaces":[{"interface":"back","average":{"1min":0.25800000000000001,"5min":0.25700000000000001,"15min":0.27400000000000002},"min":{"1min":0.192,"5min":0.17399999999999999,"15min":0.16},"max":{"1min":0.39400000000000002,"5min":0.44600000000000001,"15min":0.45100000000000001},"last":0.26500000000000001},{"interface":"front","average":{"1min":0.25900000000000001,"5min":0.26700000000000002,"15min":0.27900000000000003},"min":{"1min":0.19500000000000001,"5min":0.188,"15min":0.17499999999999999},"max":{"1min":0.41499999999999998,"5min":0.46400000000000002,"15min":0.46400000000000002},"last":0.29099999999999998}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051733,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:38:45 2023","interfaces":[{"interface":"back","average":{"1min":0.27300000000000002,"5min":0.28100000000000003,"15min":0.26400000000000001},"min":{"1min":0.19400000000000001,"5min":0.17299999999999999,"15min":0.17000000000000001},"max":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"last":0.27300000000000002},{"interface":"front","average":{"1min":0.28799999999999998,"5min":0.29099999999999998,"15min":0.27800000000000002},"min":{"1min":0.19800000000000001,"5min":0.182,"15min":0.16600000000000001},"max":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"last":0.308}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:39:54.416+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:39:54.455 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:39:54.780 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:40:09.288+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:40:09.389+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:40:10.267+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)

Ignoring malparsed backtrace: 2023-03-25T04:40:28.984 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":476,"stamp":"2023-03-25T04:40:28.245911+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002693"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":39,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:38:13.358968+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:38:13.358968+0000","last_peered":"2023-03-25T04:38:13.358968+0000","last_clean":"2023-03-25T04:38:13.358968+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:38:13.358968+0000","last_undegraded":"2023-03-25T04:38:13.358968+0000","last_fullsized":"2023-03-25T04:38:13.358968+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:38:13.358922+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:38:13.358922+0000","last_peered":"2023-03-25T04:38:13.358922+0000","last_clean":"2023-03-25T04:38:13.358922+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:38:13.358922+0000","last_undegraded":"2023-03-25T04:38:13.358922+0000","last_fullsized":"2023-03-25T04:38:13.358922+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:23.353778+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:37:23.353778+0000","last_peered":"2023-03-25T04:37:23.353778+0000","last_clean":"2023-03-25T04:37:23.353778+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:37:23.353778+0000","last_undegraded":"2023-03-25T04:37:23.353778+0000","last_fullsized":"2023-03-25T04:37:23.353778+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:40.598579+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:37:40.598579+0000","last_peered":"2023-03-25T04:37:40.598579+0000","last_clean":"2023-03-25T04:37:40.598579+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:37:40.598579+0000","last_undegraded":"2023-03-25T04:37:40.598579+0000","last_fullsized":"2023-03-25T04:37:40.598579+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":39,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:48.356344+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:37:48.356344+0000","last_peered":"2023-03-25T04:37:48.356344+0000","last_clean":"2023-03-25T04:37:48.356344+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:37:48.356344+0000","last_undegraded":"2023-03-25T04:37:48.356344+0000","last_fullsized":"2023-03-25T04:37:48.356344+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:48.356365+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:37:48.356365+0000","last_peered":"2023-03-25T04:37:48.356365+0000","last_clean":"2023-03-25T04:37:48.356365+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:37:48.356365+0000","last_undegraded":"2023-03-25T04:37:48.356365+0000","last_fullsized":"2023-03-25T04:37:48.356365+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117153,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:39:54 2023","interfaces":[{"interface":"back","average":{"1min":0.25900000000000001,"5min":0.25600000000000001,"15min":0.27100000000000002},"min":{"1min":0.188,"5min":0.17899999999999999,"15min":0.16},"max":{"1min":0.41999999999999998,"5min":0.44600000000000001,"15min":0.45100000000000001},"last":0.255},{"interface":"front","average":{"1min":0.27500000000000002,"5min":0.26600000000000001,"15min":0.27700000000000002},"min":{"1min":0.17699999999999999,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.45600000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"last":0.26800000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051743,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:39:47 2023","interfaces":[{"interface":"back","average":{"1min":0.26300000000000001,"5min":0.27700000000000002,"15min":0.26500000000000001},"min":{"1min":0.187,"5min":0.17299999999999999,"15min":0.17000000000000001},"max":{"1min":0.433,"5min":0.433,"15min":0.433},"last":0.36599999999999999},{"interface":"front","average":{"1min":0.27100000000000002,"5min":0.28799999999999998,"15min":0.27900000000000003},"min":{"1min":0.20100000000000001,"5min":0.182,"15min":0.16600000000000001},"max":{"1min":0.48599999999999999,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.34999999999999998}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:40:28.986 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean
, 2023-03-25T04:40:28.986 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}

Ignoring malparsed backtrace: 2023-03-25T04:40:47.229+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:40:47.503+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:40:48.251+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)

Ignoring malparsed backtrace: 2023-03-25T04:41:03.267 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:41:03.578 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all

Ignoring malparsed backtrace: 2023-03-25T04:41:21.347 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":502,"stamp":"2023-03-25T04:41:20.256204+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002342"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":39,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:38:13.358968+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:38:13.358968+0000","last_peered":"2023-03-25T04:38:13.358968+0000","last_clean":"2023-03-25T04:38:13.358968+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:38:13.358968+0000","last_undegraded":"2023-03-25T04:38:13.358968+0000","last_fullsized":"2023-03-25T04:38:13.358968+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:38:13.358922+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:38:13.358922+0000","last_peered":"2023-03-25T04:38:13.358922+0000","last_clean":"2023-03-25T04:38:13.358922+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:38:13.358922+0000","last_undegraded":"2023-03-25T04:38:13.358922+0000","last_fullsized":"2023-03-25T04:38:13.358922+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:23.353778+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:37:23.353778+0000","last_peered":"2023-03-25T04:37:23.353778+0000","last_clean":"2023-03-25T04:37:23.353778+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:37:23.353778+0000","last_undegraded":"2023-03-25T04:37:23.353778+0000","last_fullsized":"2023-03-25T04:37:23.353778+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:40.598579+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:37:40.598579+0000","last_peered":"2023-03-25T04:37:40.598579+0000","last_clean":"2023-03-25T04:37:40.598579+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:37:40.598579+0000","last_undegraded":"2023-03-25T04:37:40.598579+0000","last_fullsized":"2023-03-25T04:37:40.598579+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":39,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:48.356344+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:37:48.356344+0000","last_peered":"2023-03-25T04:37:48.356344+0000","last_clean":"2023-03-25T04:37:48.356344+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:37:48.356344+0000","last_undegraded":"2023-03-25T04:37:48.356344+0000","last_fullsized":"2023-03-25T04:37:48.356344+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:48.356365+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:37:48.356365+0000","last_peered":"2023-03-25T04:37:48.356365+0000","last_clean":"2023-03-25T04:37:48.356365+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:37:48.356365+0000","last_undegraded":"2023-03-25T04:37:48.356365+0000","last_fullsized":"2023-03-25T04:37:48.356365+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117163,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:40:56 2023","interfaces":[{"interface":"back","average":{"1min":0.28799999999999998,"5min":0.26300000000000001,"15min":0.27100000000000002},"min":{"1min":0.188,"5min":0.184,"15min":0.16},"max":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.17499999999999999},{"interface":"front","average":{"1min":0.29599999999999999,"5min":0.27300000000000002,"15min":0.27700000000000002},"min":{"1min":0.20599999999999999,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.43099999999999999,"5min":0.46400000000000002,"15min":0.46400000000000002},"last":0.20699999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051754,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:40:52 2023","interfaces":[{"interface":"back","average":{"1min":0.28599999999999998,"5min":0.27500000000000002,"15min":0.26700000000000002},"min":{"1min":0.17899999999999999,"5min":0.17899999999999999,"15min":0.17000000000000001},"max":{"1min":0.44,"5min":0.44,"15min":0.44},"last":0.23699999999999999},{"interface":"front","average":{"1min":0.29799999999999999,"5min":0.28699999999999998,"15min":0.28100000000000003},"min":{"1min":0.20300000000000001,"5min":0.182,"15min":0.16600000000000001},"max":{"1min":0.47599999999999998,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.28000000000000003}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:41:21.349 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean
, 2023-03-25T04:41:21.349 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}

Ignoring malparsed backtrace: 2023-03-25T04:41:39.867+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:41:40.383+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:41:40.905+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:41:58.219 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json
, 2023-03-25T04:41:58.319+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:41:58.644 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:42:18.193 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":530,"stamp":"2023-03-25T04:42:16.266659+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002356"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":39,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:38:13.358968+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:38:13.358968+0000","last_peered":"2023-03-25T04:38:13.358968+0000","last_clean":"2023-03-25T04:38:13.358968+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:38:13.358968+0000","last_undegraded":"2023-03-25T04:38:13.358968+0000","last_fullsized":"2023-03-25T04:38:13.358968+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:38:13.358922+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:38:13.358922+0000","last_peered":"2023-03-25T04:38:13.358922+0000","last_clean":"2023-03-25T04:38:13.358922+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:38:13.358922+0000","last_undegraded":"2023-03-25T04:38:13.358922+0000","last_fullsized":"2023-03-25T04:38:13.358922+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:23.364593+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:39:23.364593+0000","last_peered":"2023-03-25T04:39:23.364593+0000","last_clean":"2023-03-25T04:39:23.364593+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:39:23.364593+0000","last_undegraded":"2023-03-25T04:39:23.364593+0000","last_fullsized":"2023-03-25T04:39:23.364593+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:45.608475+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:39:45.608475+0000","last_peered":"2023-03-25T04:39:45.608475+0000","last_clean":"2023-03-25T04:39:45.608475+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:39:45.608475+0000","last_undegraded":"2023-03-25T04:39:45.608475+0000","last_fullsized":"2023-03-25T04:39:45.608475+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":40,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:53.367398+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:39:53.367398+0000","last_peered":"2023-03-25T04:39:53.367398+0000","last_clean":"2023-03-25T04:39:53.367398+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:39:53.367398+0000","last_undegraded":"2023-03-25T04:39:53.367398+0000","last_fullsized":"2023-03-25T04:39:53.367398+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:53.367424+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:39:53.367424+0000","last_peered":"2023-03-25T04:39:53.367424+0000","last_clean":"2023-03-25T04:39:53.367424+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:39:53.367424+0000","last_undegraded":"2023-03-25T04:39:53.367424+0000","last_fullsized":"2023-03-25T04:39:53.367424+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117175,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:41:58 2023","interfaces":[{"interface":"back","average":{"1min":0.25800000000000001,"5min":0.26400000000000001,"15min":0.26800000000000002},"min":{"1min":0.17499999999999999,"5min":0.17499999999999999,"15min":0.16},"max":{"1min":0.39100000000000001,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.22900000000000001},{"interface":"front","average":{"1min":0.26300000000000001,"5min":0.27200000000000002,"15min":0.27400000000000002},"min":{"1min":0.20599999999999999,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.35699999999999998,"5min":0.46400000000000002,"15min":0.46400000000000002},"last":0.26400000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051765,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:41:52 2023","interfaces":[{"interface":"back","average":{"1min":0.26500000000000001,"5min":0.27200000000000002,"15min":0.26800000000000002},"min":{"1min":0.17000000000000001,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.39200000000000002,"5min":0.44,"15min":0.44},"last":0.23899999999999999},{"interface":"front","average":{"1min":0.28100000000000003,"5min":0.28599999999999998,"15min":0.28199999999999997},"min":{"1min":0.19400000000000001,"5min":0.19400000000000001,"15min":0.16600000000000001},"max":{"1min":0.41399999999999998,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.254}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:42:18.297+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:42:18.958+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:42:35.512 INFO:teuthology.orchestra.run.smithi070.stdout:
, 2023-03-25T04:42:35.512 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":1013,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":17,"modified":"2023-03-25T04:41:54.262744+0000","services":{}},"progress_events":{}}
, 2023-03-25T04:42:35.525 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:42:52.741 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":548,"stamp":"2023-03-25T04:42:52.273601+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002221"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":40,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:40:23.370056+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:40:23.370056+0000","last_peered":"2023-03-25T04:40:23.370056+0000","last_clean":"2023-03-25T04:40:23.370056+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:40:23.370056+0000","last_undegraded":"2023-03-25T04:40:23.370056+0000","last_fullsized":"2023-03-25T04:40:23.370056+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:40:23.370039+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:40:23.370039+0000","last_peered":"2023-03-25T04:40:23.370039+0000","last_clean":"2023-03-25T04:40:23.370039+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:40:23.370039+0000","last_undegraded":"2023-03-25T04:40:23.370039+0000","last_fullsized":"2023-03-25T04:40:23.370039+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:23.364593+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:39:23.364593+0000","last_peered":"2023-03-25T04:39:23.364593+0000","last_clean":"2023-03-25T04:39:23.364593+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:39:23.364593+0000","last_undegraded":"2023-03-25T04:39:23.364593+0000","last_fullsized":"2023-03-25T04:39:23.364593+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:45.608475+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:39:45.608475+0000","last_peered":"2023-03-25T04:39:45.608475+0000","last_clean":"2023-03-25T04:39:45.608475+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:39:45.608475+0000","last_undegraded":"2023-03-25T04:39:45.608475+0000","last_fullsized":"2023-03-25T04:39:45.608475+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":40,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:53.367398+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:39:53.367398+0000","last_peered":"2023-03-25T04:39:53.367398+0000","last_clean":"2023-03-25T04:39:53.367398+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:39:53.367398+0000","last_undegraded":"2023-03-25T04:39:53.367398+0000","last_fullsized":"2023-03-25T04:39:53.367398+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:53.367424+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:39:53.367424+0000","last_peered":"2023-03-25T04:39:53.367424+0000","last_clean":"2023-03-25T04:39:53.367424+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:39:53.367424+0000","last_undegraded":"2023-03-25T04:39:53.367424+0000","last_fullsized":"2023-03-25T04:39:53.367424+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117182,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:41:58 2023","interfaces":[{"interface":"back","average":{"1min":0.25800000000000001,"5min":0.26400000000000001,"15min":0.26800000000000002},"min":{"1min":0.17499999999999999,"5min":0.17499999999999999,"15min":0.16},"max":{"1min":0.39100000000000001,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.19600000000000001},{"interface":"front","average":{"1min":0.26300000000000001,"5min":0.27200000000000002,"15min":0.27400000000000002},"min":{"1min":0.20599999999999999,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.35699999999999998,"5min":0.46400000000000002,"15min":0.46400000000000002},"last":0.219}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051773,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:41:52 2023","interfaces":[{"interface":"back","average":{"1min":0.26500000000000001,"5min":0.27200000000000002,"15min":0.26800000000000002},"min":{"1min":0.17000000000000001,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.39200000000000002,"5min":0.44,"15min":0.44},"last":0.19},{"interface":"front","average":{"1min":0.28100000000000003,"5min":0.28599999999999998,"15min":0.28199999999999997},"min":{"1min":0.19400000000000001,"5min":0.19400000000000001,"15min":0.16600000000000001},"max":{"1min":0.41399999999999998,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.17599999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:42:52.742 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean

Ignoring malparsed backtrace: 2023-03-25T04:43:09.973 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:43:09.986 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":556,"stamp":"2023-03-25T04:43:08.276556+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002359"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":40,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:40:23.370056+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:40:23.370056+0000","last_peered":"2023-03-25T04:40:23.370056+0000","last_clean":"2023-03-25T04:40:23.370056+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:40:23.370056+0000","last_undegraded":"2023-03-25T04:40:23.370056+0000","last_fullsized":"2023-03-25T04:40:23.370056+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:40:23.370039+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:40:23.370039+0000","last_peered":"2023-03-25T04:40:23.370039+0000","last_clean":"2023-03-25T04:40:23.370039+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:40:23.370039+0000","last_undegraded":"2023-03-25T04:40:23.370039+0000","last_fullsized":"2023-03-25T04:40:23.370039+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:23.364593+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:39:23.364593+0000","last_peered":"2023-03-25T04:39:23.364593+0000","last_clean":"2023-03-25T04:39:23.364593+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:39:23.364593+0000","last_undegraded":"2023-03-25T04:39:23.364593+0000","last_fullsized":"2023-03-25T04:39:23.364593+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:45.608475+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:39:45.608475+0000","last_peered":"2023-03-25T04:39:45.608475+0000","last_clean":"2023-03-25T04:39:45.608475+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:39:45.608475+0000","last_undegraded":"2023-03-25T04:39:45.608475+0000","last_fullsized":"2023-03-25T04:39:45.608475+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":40,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:53.367398+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:39:53.367398+0000","last_peered":"2023-03-25T04:39:53.367398+0000","last_clean":"2023-03-25T04:39:53.367398+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:39:53.367398+0000","last_undegraded":"2023-03-25T04:39:53.367398+0000","last_fullsized":"2023-03-25T04:39:53.367398+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:53.367424+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:39:53.367424+0000","last_peered":"2023-03-25T04:39:53.367424+0000","last_clean":"2023-03-25T04:39:53.367424+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:39:53.367424+0000","last_undegraded":"2023-03-25T04:39:53.367424+0000","last_fullsized":"2023-03-25T04:39:53.367424+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117185,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:43:00 2023","interfaces":[{"interface":"back","average":{"1min":0.27400000000000002,"5min":0.26700000000000002,"15min":0.26600000000000001},"min":{"1min":0.19600000000000001,"5min":0.17499999999999999,"15min":0.16},"max":{"1min":0.42999999999999999,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.27800000000000002},{"interface":"front","average":{"1min":0.28199999999999997,"5min":0.27500000000000002,"15min":0.27300000000000002},"min":{"1min":0.185,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.47199999999999998,"5min":0.47199999999999998,"15min":0.47199999999999998},"last":0.247}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051776,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:42:54 2023","interfaces":[{"interface":"back","average":{"1min":0.26300000000000001,"5min":0.27000000000000002,"15min":0.26900000000000002},"min":{"1min":0.17399999999999999,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.438,"5min":0.44,"15min":0.44},"last":0.20100000000000001},{"interface":"front","average":{"1min":0.27400000000000002,"5min":0.28199999999999997,"15min":0.28299999999999997},"min":{"1min":0.17599999999999999,"5min":0.17599999999999999,"15min":0.16600000000000001},"max":{"1min":0.41599999999999998,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.23499999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:43:25.042+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:43:25.411+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:43:26.013+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:43:47.937+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:43:48.534+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:43:48.889+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:44:09.318 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:44:09.331 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":586,"stamp":"2023-03-25T04:44:08.288933+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002697"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":40,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:40:23.370056+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:40:23.370056+0000","last_peered":"2023-03-25T04:40:23.370056+0000","last_clean":"2023-03-25T04:40:23.370056+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:40:23.370056+0000","last_undegraded":"2023-03-25T04:40:23.370056+0000","last_fullsized":"2023-03-25T04:40:23.370056+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:40:23.370039+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:40:23.370039+0000","last_peered":"2023-03-25T04:40:23.370039+0000","last_clean":"2023-03-25T04:40:23.370039+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:40:23.370039+0000","last_undegraded":"2023-03-25T04:40:23.370039+0000","last_fullsized":"2023-03-25T04:40:23.370039+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:23.375106+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:41:23.375106+0000","last_peered":"2023-03-25T04:41:23.375106+0000","last_clean":"2023-03-25T04:41:23.375106+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:41:23.375106+0000","last_undegraded":"2023-03-25T04:41:23.375106+0000","last_fullsized":"2023-03-25T04:41:23.375106+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:50.618930+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:41:50.618930+0000","last_peered":"2023-03-25T04:41:50.618930+0000","last_clean":"2023-03-25T04:41:50.618930+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:41:50.618930+0000","last_undegraded":"2023-03-25T04:41:50.618930+0000","last_fullsized":"2023-03-25T04:41:50.618930+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":41,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:58.377821+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:41:58.377821+0000","last_peered":"2023-03-25T04:41:58.377821+0000","last_clean":"2023-03-25T04:41:58.377821+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:41:58.377821+0000","last_undegraded":"2023-03-25T04:41:58.377821+0000","last_fullsized":"2023-03-25T04:41:58.377821+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:58.377836+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:41:58.377836+0000","last_peered":"2023-03-25T04:41:58.377836+0000","last_clean":"2023-03-25T04:41:58.377836+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:41:58.377836+0000","last_undegraded":"2023-03-25T04:41:58.377836+0000","last_fullsized":"2023-03-25T04:41:58.377836+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117197,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:44:01 2023","interfaces":[{"interface":"back","average":{"1min":0.27200000000000002,"5min":0.27000000000000002,"15min":0.26400000000000001},"min":{"1min":0.20399999999999999,"5min":0.17499999999999999,"15min":0.16},"max":{"1min":0.38400000000000001,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.245},{"interface":"front","average":{"1min":0.27600000000000002,"5min":0.27800000000000002,"15min":0.27100000000000002},"min":{"1min":0.23100000000000001,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.40400000000000003,"5min":0.47199999999999998,"15min":0.47199999999999998},"last":0.27500000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051788,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:43:55 2023","interfaces":[{"interface":"back","average":{"1min":0.26100000000000001,"5min":0.26700000000000002,"15min":0.27000000000000002},"min":{"1min":0.17799999999999999,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.374,"5min":0.44,"15min":0.44},"last":0.30299999999999999},{"interface":"front","average":{"1min":0.28299999999999997,"5min":0.28100000000000003,"15min":0.28399999999999997},"min":{"1min":0.20899999999999999,"5min":0.17599999999999999,"15min":0.16600000000000001},"max":{"1min":0.40100000000000002,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.27800000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:44:28.923 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":1126,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":19,"modified":"2023-03-25T04:43:20.278959+0000","services":{}},"progress_events":{}}
, 2023-03-25T04:44:28.936 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now
, 2023-03-25T04:44:28.937 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace: 2023-03-25T04:44:48.786 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:44:48.800 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":606,"stamp":"2023-03-25T04:44:48.296805+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002182"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":41,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:42:34.382196+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:42:34.382196+0000","last_peered":"2023-03-25T04:42:34.382196+0000","last_clean":"2023-03-25T04:42:34.382196+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:42:34.382196+0000","last_undegraded":"2023-03-25T04:42:34.382196+0000","last_fullsized":"2023-03-25T04:42:34.382196+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:42:29.381744+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:42:29.381744+0000","last_peered":"2023-03-25T04:42:29.381744+0000","last_clean":"2023-03-25T04:42:29.381744+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:42:29.381744+0000","last_undegraded":"2023-03-25T04:42:29.381744+0000","last_fullsized":"2023-03-25T04:42:29.381744+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:23.375106+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:41:23.375106+0000","last_peered":"2023-03-25T04:41:23.375106+0000","last_clean":"2023-03-25T04:41:23.375106+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:41:23.375106+0000","last_undegraded":"2023-03-25T04:41:23.375106+0000","last_fullsized":"2023-03-25T04:41:23.375106+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:50.618930+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:41:50.618930+0000","last_peered":"2023-03-25T04:41:50.618930+0000","last_clean":"2023-03-25T04:41:50.618930+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:41:50.618930+0000","last_undegraded":"2023-03-25T04:41:50.618930+0000","last_fullsized":"2023-03-25T04:41:50.618930+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":41,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:58.377821+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:41:58.377821+0000","last_peered":"2023-03-25T04:41:58.377821+0000","last_clean":"2023-03-25T04:41:58.377821+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:41:58.377821+0000","last_undegraded":"2023-03-25T04:41:58.377821+0000","last_fullsized":"2023-03-25T04:41:58.377821+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:58.377836+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:41:58.377836+0000","last_peered":"2023-03-25T04:41:58.377836+0000","last_clean":"2023-03-25T04:41:58.377836+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:41:58.377836+0000","last_undegraded":"2023-03-25T04:41:58.377836+0000","last_fullsized":"2023-03-25T04:41:58.377836+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117206,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:44:01 2023","interfaces":[{"interface":"back","average":{"1min":0.27200000000000002,"5min":0.27000000000000002,"15min":0.26400000000000001},"min":{"1min":0.20399999999999999,"5min":0.17499999999999999,"15min":0.16},"max":{"1min":0.38400000000000001,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.216},{"interface":"front","average":{"1min":0.27600000000000002,"5min":0.27800000000000002,"15min":0.27100000000000002},"min":{"1min":0.23100000000000001,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.40400000000000003,"5min":0.47199999999999998,"15min":0.47199999999999998},"last":0.23999999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051796,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:43:55 2023","interfaces":[{"interface":"back","average":{"1min":0.26100000000000001,"5min":0.26700000000000002,"15min":0.27000000000000002},"min":{"1min":0.17799999999999999,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.374,"5min":0.44,"15min":0.44},"last":0.25600000000000001},{"interface":"front","average":{"1min":0.28299999999999997,"5min":0.28100000000000003,"15min":0.28399999999999997},"min":{"1min":0.20899999999999999,"5min":0.17599999999999999,"15min":0.16600000000000001},"max":{"1min":0.40100000000000002,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.28000000000000003}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:45:05.762 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:45:06.073 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all

Ignoring malparsed backtrace: 2023-03-25T04:45:25.658 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":624,"stamp":"2023-03-25T04:45:24.304294+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002366"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":41,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:42:34.382196+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:42:34.382196+0000","last_peered":"2023-03-25T04:42:34.382196+0000","last_clean":"2023-03-25T04:42:34.382196+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:42:34.382196+0000","last_undegraded":"2023-03-25T04:42:34.382196+0000","last_fullsized":"2023-03-25T04:42:34.382196+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:42:29.381744+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:42:29.381744+0000","last_peered":"2023-03-25T04:42:29.381744+0000","last_clean":"2023-03-25T04:42:29.381744+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:42:29.381744+0000","last_undegraded":"2023-03-25T04:42:29.381744+0000","last_fullsized":"2023-03-25T04:42:29.381744+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:23.375106+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:41:23.375106+0000","last_peered":"2023-03-25T04:41:23.375106+0000","last_clean":"2023-03-25T04:41:23.375106+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:41:23.375106+0000","last_undegraded":"2023-03-25T04:41:23.375106+0000","last_fullsized":"2023-03-25T04:41:23.375106+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:50.618930+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:41:50.618930+0000","last_peered":"2023-03-25T04:41:50.618930+0000","last_clean":"2023-03-25T04:41:50.618930+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:41:50.618930+0000","last_undegraded":"2023-03-25T04:41:50.618930+0000","last_fullsized":"2023-03-25T04:41:50.618930+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":41,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:58.377821+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:41:58.377821+0000","last_peered":"2023-03-25T04:41:58.377821+0000","last_clean":"2023-03-25T04:41:58.377821+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:41:58.377821+0000","last_undegraded":"2023-03-25T04:41:58.377821+0000","last_fullsized":"2023-03-25T04:41:58.377821+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:58.377836+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:41:58.377836+0000","last_peered":"2023-03-25T04:41:58.377836+0000","last_clean":"2023-03-25T04:41:58.377836+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:41:58.377836+0000","last_undegraded":"2023-03-25T04:41:58.377836+0000","last_fullsized":"2023-03-25T04:41:58.377836+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117213,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:45:01 2023","interfaces":[{"interface":"back","average":{"1min":0.251,"5min":0.26800000000000002,"15min":0.26400000000000001},"min":{"1min":0.183,"5min":0.17499999999999999,"15min":0.16},"max":{"1min":0.41499999999999998,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.26900000000000002},{"interface":"front","average":{"1min":0.28000000000000003,"5min":0.27900000000000003,"15min":0.27100000000000002},"min":{"1min":0.20999999999999999,"5min":0.185,"15min":0.17499999999999999},"max":{"1min":0.44400000000000001,"5min":0.47199999999999998,"15min":0.47199999999999998},"last":0.27900000000000003}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051803,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:44:55 2023","interfaces":[{"interface":"back","average":{"1min":0.252,"5min":0.26500000000000001,"15min":0.26900000000000002},"min":{"1min":0.183,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.39700000000000002,"5min":0.44,"15min":0.44},"last":0.20699999999999999},{"interface":"front","average":{"1min":0.27000000000000002,"5min":0.28100000000000003,"15min":0.28299999999999997},"min":{"1min":0.20300000000000001,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":0.42099999999999999,"5min":0.47599999999999998,"15min":0.48599999999999999},"last":0.23599999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:45:25.659 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:45:25.969 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:45:42.735+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:45:42.744+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:45:42.897 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace: 2023-03-25T04:46:00.183 INFO:teuthology.orchestra.run.smithi070.stdout:
, 2023-03-25T04:46:00.183 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":1217,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":20,"modified":"2023-03-25T04:45:42.307700+0000","services":{"mgr":{"daemons":{"summary":"","x":{"start_epoch":0,"start_stamp":"0.000000","gid":0,"addr":"(unrecognized address family 0)/0","metadata":{},"task_status":{}}}}}},"progress_events":{}}
, 2023-03-25T04:46:00.197 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now

Ignoring malparsed backtrace: 2023-03-25T04:46:19.910+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:46:20.039 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all

Ignoring malparsed backtrace: 2023-03-25T04:46:39.611 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}
, 2023-03-25T04:46:39.611 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json
, 2023-03-25T04:46:39.613+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:46:58.485+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:46:58.764+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:46:59.519+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:47:14.132 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:47:14.146 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":678,"stamp":"2023-03-25T04:47:12.325044+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002571"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":42,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:44.393426+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:44:44.393426+0000","last_peered":"2023-03-25T04:44:44.393426+0000","last_clean":"2023-03-25T04:44:44.393426+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:44:44.393426+0000","last_undegraded":"2023-03-25T04:44:44.393426+0000","last_fullsized":"2023-03-25T04:44:44.393426+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:39.393044+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:44:39.393044+0000","last_peered":"2023-03-25T04:44:39.393044+0000","last_clean":"2023-03-25T04:44:39.393044+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:44:39.393044+0000","last_undegraded":"2023-03-25T04:44:39.393044+0000","last_fullsized":"2023-03-25T04:44:39.393044+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:43:24.386177+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:43:24.386177+0000","last_peered":"2023-03-25T04:43:24.386177+0000","last_clean":"2023-03-25T04:43:24.386177+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:43:24.386177+0000","last_undegraded":"2023-03-25T04:43:24.386177+0000","last_fullsized":"2023-03-25T04:43:24.386177+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:43:55.629798+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:43:55.629798+0000","last_peered":"2023-03-25T04:43:55.629798+0000","last_clean":"2023-03-25T04:43:55.629798+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:43:55.629798+0000","last_undegraded":"2023-03-25T04:43:55.629798+0000","last_fullsized":"2023-03-25T04:43:55.629798+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":42,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:04.389904+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:44:04.389904+0000","last_peered":"2023-03-25T04:44:04.389904+0000","last_clean":"2023-03-25T04:44:04.389904+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:44:04.389904+0000","last_undegraded":"2023-03-25T04:44:04.389904+0000","last_fullsized":"2023-03-25T04:44:04.389904+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:04.389923+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:44:04.389923+0000","last_peered":"2023-03-25T04:44:04.389923+0000","last_clean":"2023-03-25T04:44:04.389923+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:44:04.389923+0000","last_undegraded":"2023-03-25T04:44:04.389923+0000","last_fullsized":"2023-03-25T04:44:04.389923+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117235,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:47:05 2023","interfaces":[{"interface":"back","average":{"1min":0.27800000000000002,"5min":0.27000000000000002,"15min":0.26300000000000001},"min":{"1min":0.20100000000000001,"5min":0.16900000000000001,"15min":0.16},"max":{"1min":0.39200000000000002,"5min":0.42999999999999999,"15min":0.45600000000000002},"last":0.379},{"interface":"front","average":{"1min":0.28399999999999997,"5min":0.28100000000000003,"15min":0.27200000000000002},"min":{"1min":0.17799999999999999,"5min":0.17799999999999999,"15min":0.17699999999999999},"max":{"1min":0.42599999999999999,"5min":0.47199999999999998,"15min":0.47199999999999998},"last":0.39800000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051825,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:47:01 2023","interfaces":[{"interface":"back","average":{"1min":0.27300000000000002,"5min":0.26100000000000001,"15min":0.27000000000000002},"min":{"1min":0.186,"5min":0.17399999999999999,"15min":0.17000000000000001},"max":{"1min":0.40400000000000003,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.23899999999999999},{"interface":"front","average":{"1min":0.28699999999999998,"5min":0.27500000000000002,"15min":0.28299999999999997},"min":{"1min":0.186,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.48599999999999999},"last":0.19600000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:47:31.198 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json
, 2023-03-25T04:47:31.411+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:47:31.789 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:47:49.509 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":696,"stamp":"2023-03-25T04:47:48.332093+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002697"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":42,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:44.393426+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:44:44.393426+0000","last_peered":"2023-03-25T04:44:44.393426+0000","last_clean":"2023-03-25T04:44:44.393426+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:44:44.393426+0000","last_undegraded":"2023-03-25T04:44:44.393426+0000","last_fullsized":"2023-03-25T04:44:44.393426+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:39.393044+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:44:39.393044+0000","last_peered":"2023-03-25T04:44:39.393044+0000","last_clean":"2023-03-25T04:44:39.393044+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:44:39.393044+0000","last_undegraded":"2023-03-25T04:44:39.393044+0000","last_fullsized":"2023-03-25T04:44:39.393044+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:45:24.396717+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:45:24.396717+0000","last_peered":"2023-03-25T04:45:24.396717+0000","last_clean":"2023-03-25T04:45:24.396717+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:45:24.396717+0000","last_undegraded":"2023-03-25T04:45:24.396717+0000","last_fullsized":"2023-03-25T04:45:24.396717+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:43:55.629798+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:43:55.629798+0000","last_peered":"2023-03-25T04:43:55.629798+0000","last_clean":"2023-03-25T04:43:55.629798+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:43:55.629798+0000","last_undegraded":"2023-03-25T04:43:55.629798+0000","last_fullsized":"2023-03-25T04:43:55.629798+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":42,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:04.389904+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:44:04.389904+0000","last_peered":"2023-03-25T04:44:04.389904+0000","last_clean":"2023-03-25T04:44:04.389904+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:44:04.389904+0000","last_undegraded":"2023-03-25T04:44:04.389904+0000","last_fullsized":"2023-03-25T04:44:04.389904+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:04.389923+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:44:04.389923+0000","last_peered":"2023-03-25T04:44:04.389923+0000","last_clean":"2023-03-25T04:44:04.389923+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:44:04.389923+0000","last_undegraded":"2023-03-25T04:44:04.389923+0000","last_fullsized":"2023-03-25T04:44:04.389923+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117242,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:47:05 2023","interfaces":[{"interface":"back","average":{"1min":0.27800000000000002,"5min":0.27000000000000002,"15min":0.26300000000000001},"min":{"1min":0.20100000000000001,"5min":0.16900000000000001,"15min":0.16},"max":{"1min":0.39200000000000002,"5min":0.42999999999999999,"15min":0.45600000000000002},"last":0.40500000000000003},{"interface":"front","average":{"1min":0.28399999999999997,"5min":0.28100000000000003,"15min":0.27200000000000002},"min":{"1min":0.17799999999999999,"5min":0.17799999999999999,"15min":0.17699999999999999},"max":{"1min":0.42599999999999999,"5min":0.47199999999999998,"15min":0.47199999999999998},"last":0.42399999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051834,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:47:01 2023","interfaces":[{"interface":"back","average":{"1min":0.27300000000000002,"5min":0.26100000000000001,"15min":0.27000000000000002},"min":{"1min":0.186,"5min":0.17399999999999999,"15min":0.17000000000000001},"max":{"1min":0.40400000000000003,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.25600000000000001},{"interface":"front","average":{"1min":0.28699999999999998,"5min":0.27500000000000002,"15min":0.28299999999999997},"min":{"1min":0.186,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.48599999999999999},"last":0.22800000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:47:49.510 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace:     ctx.managers[config['cluster']].wait_for_clean()
,   File "/home/teuthworker/src/github.com_ceph_ceph-c_511015c23eef0684a64171b710e432448df2b7c7/qa/tasks/ceph_manager.py", line 2747, in wait_for_clean
,     assert time.time() - start < timeout, \

Ignoring malparsed backtrace:  ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
,  1: /lib64/libpthread.so.0(+0x12cf0) [0x7f1453c2fcf0]
,  2: __pthread_mutex_lock()

Ignoring malparsed backtrace: 2023-03-25T04:28:05.172 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":142,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":5,"modified":"2023-03-25T04:27:12.092098+0000","services":{}},"progress_events":{}}
, 2023-03-25T04:28:05.185 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now
, 2023-03-25T04:28:05.186 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace: 2023-03-25T04:28:28.360 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":116,"stamp":"2023-03-25T04:28:28.108007+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002578"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.314661+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:27:45.314661+0000","last_peered":"2023-03-25T04:27:45.314661+0000","last_clean":"2023-03-25T04:27:45.314661+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:27:45.314661+0000","last_undegraded":"2023-03-25T04:27:45.314661+0000","last_fullsized":"2023-03-25T04:27:45.314661+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":25,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.310745+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:27:44.310745+0000","last_peered":"2023-03-25T04:27:44.310745+0000","last_clean":"2023-03-25T04:27:44.310745+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:27:44.310745+0000","last_undegraded":"2023-03-25T04:27:44.310745+0000","last_fullsized":"2023-03-25T04:27:44.310745+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":25,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.310650+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:27:44.310650+0000","last_peered":"2023-03-25T04:27:44.310650+0000","last_clean":"2023-03-25T04:27:44.310650+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:27:44.310650+0000","last_undegraded":"2023-03-25T04:27:44.310650+0000","last_fullsized":"2023-03-25T04:27:44.310650+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":26,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.312962+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:27:44.312962+0000","last_peered":"2023-03-25T04:27:44.312962+0000","last_clean":"2023-03-25T04:27:44.312962+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:27:44.312962+0000","last_undegraded":"2023-03-25T04:27:44.312962+0000","last_fullsized":"2023-03-25T04:27:44.312962+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315064+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:27:45.315064+0000","last_peered":"2023-03-25T04:27:45.315064+0000","last_clean":"2023-03-25T04:27:45.315064+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:27:45.315064+0000","last_undegraded":"2023-03-25T04:27:45.315064+0000","last_fullsized":"2023-03-25T04:27:45.315064+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":25,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.310707+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:27:44.310707+0000","last_peered":"2023-03-25T04:27:44.310707+0000","last_clean":"2023-03-25T04:27:44.310707+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:27:44.310707+0000","last_undegraded":"2023-03-25T04:27:44.310707+0000","last_fullsized":"2023-03-25T04:27:44.310707+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117007,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.375}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051597,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.186}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:28:28.362 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean
, 2023-03-25T04:28:28.362 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:28:45.655 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":124,"stamp":"2023-03-25T04:28:44.111265+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002492"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.314661+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:27:45.314661+0000","last_peered":"2023-03-25T04:27:45.314661+0000","last_clean":"2023-03-25T04:27:45.314661+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:27:45.314661+0000","last_undegraded":"2023-03-25T04:27:45.314661+0000","last_fullsized":"2023-03-25T04:27:45.314661+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":25,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.310745+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:27:44.310745+0000","last_peered":"2023-03-25T04:27:44.310745+0000","last_clean":"2023-03-25T04:27:44.310745+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:27:44.310745+0000","last_undegraded":"2023-03-25T04:27:44.310745+0000","last_fullsized":"2023-03-25T04:27:44.310745+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":25,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.310650+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:27:44.310650+0000","last_peered":"2023-03-25T04:27:44.310650+0000","last_clean":"2023-03-25T04:27:44.310650+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:27:44.310650+0000","last_undegraded":"2023-03-25T04:27:44.310650+0000","last_fullsized":"2023-03-25T04:27:44.310650+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":26,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.312962+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:27:44.312962+0000","last_peered":"2023-03-25T04:27:44.312962+0000","last_clean":"2023-03-25T04:27:44.312962+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:27:44.312962+0000","last_undegraded":"2023-03-25T04:27:44.312962+0000","last_fullsized":"2023-03-25T04:27:44.312962+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315064+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:27:45.315064+0000","last_peered":"2023-03-25T04:27:45.315064+0000","last_clean":"2023-03-25T04:27:45.315064+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:27:45.315064+0000","last_undegraded":"2023-03-25T04:27:45.315064+0000","last_fullsized":"2023-03-25T04:27:45.315064+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":25,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.310707+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:27:44.310707+0000","last_peered":"2023-03-25T04:27:44.310707+0000","last_clean":"2023-03-25T04:27:44.310707+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:27:44.310707+0000","last_undegraded":"2023-03-25T04:27:44.310707+0000","last_fullsized":"2023-03-25T04:27:44.310707+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117011,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:28:31 2023","interfaces":[{"interface":"back","average":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.36899999999999999},{"interface":"front","average":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"min":{"1min":0.19,"5min":0.19,"15min":0.19},"max":{"1min":0.46300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.39800000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051601,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:28:24 2023","interfaces":[{"interface":"back","average":{"1min":0.249,"5min":0.249,"15min":0.249},"min":{"1min":0.17699999999999999,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"last":0.254},{"interface":"front","average":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"min":{"1min":0.20200000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"last":0.312}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:28:45.656 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace: 2023-03-25T04:29:05.171 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}
, 2023-03-25T04:29:05.171 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json
, 2023-03-25T04:29:05.561 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:29:22.465 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:29:22.773 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all

Ignoring malparsed backtrace: 2023-03-25T04:29:42.405 INFO:teuthology.orchestra.run.smithi070.stdout:
, 2023-03-25T04:29:42.406 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":240,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":9,"modified":"2023-03-25T04:29:32.120358+0000","services":{"mon":{"daemons":{"summary":"","a":{"start_epoch":0,"start_stamp":"0.000000","gid":0,"addr":"(unrecognized address family 0)/0","metadata":{},"task_status":{}}}}}},"progress_events":{}}
, 2023-03-25T04:29:42.419 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:29:59.627 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":161,"stamp":"2023-03-25T04:29:58.125186+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002398"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315468+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:27:45.315468+0000","last_peered":"2023-03-25T04:27:45.315468+0000","last_clean":"2023-03-25T04:27:45.315468+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:27:45.315468+0000","last_undegraded":"2023-03-25T04:27:45.315468+0000","last_fullsized":"2023-03-25T04:27:45.315468+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312845+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:27:45.312845+0000","last_peered":"2023-03-25T04:27:45.312845+0000","last_clean":"2023-03-25T04:27:45.312845+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:27:45.312845+0000","last_undegraded":"2023-03-25T04:27:45.312845+0000","last_fullsized":"2023-03-25T04:27:45.312845+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312882+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:27:45.312882+0000","last_peered":"2023-03-25T04:27:45.312882+0000","last_clean":"2023-03-25T04:27:45.312882+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:27:45.312882+0000","last_undegraded":"2023-03-25T04:27:45.312882+0000","last_fullsized":"2023-03-25T04:27:45.312882+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.311730+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:27:45.311730+0000","last_peered":"2023-03-25T04:27:45.311730+0000","last_clean":"2023-03-25T04:27:45.311730+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:27:45.311730+0000","last_undegraded":"2023-03-25T04:27:45.311730+0000","last_fullsized":"2023-03-25T04:27:45.311730+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315735+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:27:45.315735+0000","last_peered":"2023-03-25T04:27:45.315735+0000","last_clean":"2023-03-25T04:27:45.315735+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:27:45.315735+0000","last_undegraded":"2023-03-25T04:27:45.315735+0000","last_fullsized":"2023-03-25T04:27:45.315735+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312833+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:27:45.312833+0000","last_peered":"2023-03-25T04:27:45.312833+0000","last_clean":"2023-03-25T04:27:45.312833+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:27:45.312833+0000","last_undegraded":"2023-03-25T04:27:45.312833+0000","last_fullsized":"2023-03-25T04:27:45.312833+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117026,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:29:33 2023","interfaces":[{"interface":"back","average":{"1min":0.26300000000000001,"5min":0.29099999999999998,"15min":0.29599999999999999},"min":{"1min":0.191,"5min":0.191,"15min":0.191},"max":{"1min":0.371,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.20499999999999999},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.29599999999999999,"15min":0.29999999999999999},"min":{"1min":0.21099999999999999,"5min":0.19,"15min":0.19},"max":{"1min":0.39800000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.19500000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051616,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:29:29 2023","interfaces":[{"interface":"back","average":{"1min":0.27200000000000002,"5min":0.253,"15min":0.25},"min":{"1min":0.183,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"last":0.26400000000000001},{"interface":"front","average":{"1min":0.28499999999999998,"5min":0.26800000000000002,"15min":0.26500000000000001},"min":{"1min":0.16600000000000001,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"last":0.29899999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:29:59.629 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean

Ignoring malparsed backtrace: 2023-03-25T04:30:16.889 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:30:16.903 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":170,"stamp":"2023-03-25T04:30:16.128427+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002222"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315468+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:27:45.315468+0000","last_peered":"2023-03-25T04:27:45.315468+0000","last_clean":"2023-03-25T04:27:45.315468+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:27:45.315468+0000","last_undegraded":"2023-03-25T04:27:45.315468+0000","last_fullsized":"2023-03-25T04:27:45.315468+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312845+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:27:45.312845+0000","last_peered":"2023-03-25T04:27:45.312845+0000","last_clean":"2023-03-25T04:27:45.312845+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:27:45.312845+0000","last_undegraded":"2023-03-25T04:27:45.312845+0000","last_fullsized":"2023-03-25T04:27:45.312845+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312882+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:27:45.312882+0000","last_peered":"2023-03-25T04:27:45.312882+0000","last_clean":"2023-03-25T04:27:45.312882+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:27:45.312882+0000","last_undegraded":"2023-03-25T04:27:45.312882+0000","last_fullsized":"2023-03-25T04:27:45.312882+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.311730+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:27:45.311730+0000","last_peered":"2023-03-25T04:27:45.311730+0000","last_clean":"2023-03-25T04:27:45.311730+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:27:45.311730+0000","last_undegraded":"2023-03-25T04:27:45.311730+0000","last_fullsized":"2023-03-25T04:27:45.311730+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315735+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:27:45.315735+0000","last_peered":"2023-03-25T04:27:45.315735+0000","last_clean":"2023-03-25T04:27:45.315735+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:27:45.315735+0000","last_undegraded":"2023-03-25T04:27:45.315735+0000","last_fullsized":"2023-03-25T04:27:45.315735+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312833+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:27:45.312833+0000","last_peered":"2023-03-25T04:27:45.312833+0000","last_clean":"2023-03-25T04:27:45.312833+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:27:45.312833+0000","last_undegraded":"2023-03-25T04:27:45.312833+0000","last_fullsized":"2023-03-25T04:27:45.312833+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117030,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:29:33 2023","interfaces":[{"interface":"back","average":{"1min":0.26300000000000001,"5min":0.29099999999999998,"15min":0.29599999999999999},"min":{"1min":0.191,"5min":0.191,"15min":0.191},"max":{"1min":0.371,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.245},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.29599999999999999,"15min":0.29999999999999999},"min":{"1min":0.21099999999999999,"5min":0.19,"15min":0.19},"max":{"1min":0.39800000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.20899999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051620,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:29:29 2023","interfaces":[{"interface":"back","average":{"1min":0.27200000000000002,"5min":0.253,"15min":0.25},"min":{"1min":0.183,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"last":0.19400000000000001},{"interface":"front","average":{"1min":0.28499999999999998,"5min":0.26800000000000002,"15min":0.26500000000000001},"min":{"1min":0.16600000000000001,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"last":0.23599999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:30:31.984+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:30:32.201+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:30:32.981+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)

Ignoring malparsed backtrace: 2023-03-25T04:30:48.449 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now
, 2023-03-25T04:30:48.450 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:30:48.760 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:31:08.342 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":196,"stamp":"2023-03-25T04:31:08.139018+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002431"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315468+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:27:45.315468+0000","last_peered":"2023-03-25T04:27:45.315468+0000","last_clean":"2023-03-25T04:27:45.315468+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:27:45.315468+0000","last_undegraded":"2023-03-25T04:27:45.315468+0000","last_fullsized":"2023-03-25T04:27:45.315468+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312845+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:27:45.312845+0000","last_peered":"2023-03-25T04:27:45.312845+0000","last_clean":"2023-03-25T04:27:45.312845+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:27:45.312845+0000","last_undegraded":"2023-03-25T04:27:45.312845+0000","last_fullsized":"2023-03-25T04:27:45.312845+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312882+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:27:45.312882+0000","last_peered":"2023-03-25T04:27:45.312882+0000","last_clean":"2023-03-25T04:27:45.312882+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:27:45.312882+0000","last_undegraded":"2023-03-25T04:27:45.312882+0000","last_fullsized":"2023-03-25T04:27:45.312882+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.311730+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:27:45.311730+0000","last_peered":"2023-03-25T04:27:45.311730+0000","last_clean":"2023-03-25T04:27:45.311730+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:27:45.311730+0000","last_undegraded":"2023-03-25T04:27:45.311730+0000","last_fullsized":"2023-03-25T04:27:45.311730+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315735+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:27:45.315735+0000","last_peered":"2023-03-25T04:27:45.315735+0000","last_clean":"2023-03-25T04:27:45.315735+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:27:45.315735+0000","last_undegraded":"2023-03-25T04:27:45.315735+0000","last_fullsized":"2023-03-25T04:27:45.315735+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312833+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:27:45.312833+0000","last_peered":"2023-03-25T04:27:45.312833+0000","last_clean":"2023-03-25T04:27:45.312833+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:27:45.312833+0000","last_undegraded":"2023-03-25T04:27:45.312833+0000","last_fullsized":"2023-03-25T04:27:45.312833+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117040,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:30:37 2023","interfaces":[{"interface":"back","average":{"1min":0.28799999999999998,"5min":0.28899999999999998,"15min":0.29499999999999998},"min":{"1min":0.20200000000000001,"5min":0.191,"15min":0.191},"max":{"1min":0.433,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.29799999999999999},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.28999999999999998,"15min":0.29799999999999999},"min":{"1min":0.17499999999999999,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":0.40300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.25700000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051630,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:30:32 2023","interfaces":[{"interface":"back","average":{"1min":0.25600000000000001,"5min":0.255,"15min":0.251},"min":{"1min":0.19400000000000001,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.35699999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"last":0.22800000000000001},{"interface":"front","average":{"1min":0.28699999999999998,"5min":0.27200000000000002,"15min":0.26600000000000001},"min":{"1min":0.20599999999999999,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.39400000000000002,"5min":0.42899999999999999,"15min":0.42899999999999999},"last":0.24099999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:31:08.343 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace: 2023-03-25T04:31:24.283+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:31:25.192+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:31:25.295+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)

Ignoring malparsed backtrace: 2023-03-25T04:31:40.257 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:31:40.271 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":212,"stamp":"2023-03-25T04:31:40.144732+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002208"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315468+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:27:45.315468+0000","last_peered":"2023-03-25T04:27:45.315468+0000","last_clean":"2023-03-25T04:27:45.315468+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:27:45.315468+0000","last_undegraded":"2023-03-25T04:27:45.315468+0000","last_fullsized":"2023-03-25T04:27:45.315468+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312845+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:27:45.312845+0000","last_peered":"2023-03-25T04:27:45.312845+0000","last_clean":"2023-03-25T04:27:45.312845+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:27:45.312845+0000","last_undegraded":"2023-03-25T04:27:45.312845+0000","last_fullsized":"2023-03-25T04:27:45.312845+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:23.308607+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:29:23.308607+0000","last_peered":"2023-03-25T04:29:23.308607+0000","last_clean":"2023-03-25T04:29:23.308607+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:29:23.308607+0000","last_undegraded":"2023-03-25T04:29:23.308607+0000","last_fullsized":"2023-03-25T04:29:23.308607+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":28,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:20.554710+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:29:20.554710+0000","last_peered":"2023-03-25T04:29:20.554710+0000","last_clean":"2023-03-25T04:29:20.554710+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:29:20.554710+0000","last_undegraded":"2023-03-25T04:29:20.554710+0000","last_fullsized":"2023-03-25T04:29:20.554710+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309216+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:29:28.309216+0000","last_peered":"2023-03-25T04:29:28.309216+0000","last_clean":"2023-03-25T04:29:28.309216+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:29:28.309216+0000","last_undegraded":"2023-03-25T04:29:28.309216+0000","last_fullsized":"2023-03-25T04:29:28.309216+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309233+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:29:28.309233+0000","last_peered":"2023-03-25T04:29:28.309233+0000","last_clean":"2023-03-25T04:29:28.309233+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:29:28.309233+0000","last_undegraded":"2023-03-25T04:29:28.309233+0000","last_fullsized":"2023-03-25T04:29:28.309233+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117046,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:30:37 2023","interfaces":[{"interface":"back","average":{"1min":0.28799999999999998,"5min":0.28899999999999998,"15min":0.29499999999999998},"min":{"1min":0.20200000000000001,"5min":0.191,"15min":0.191},"max":{"1min":0.433,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.216},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.28999999999999998,"15min":0.29799999999999999},"min":{"1min":0.17499999999999999,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":0.40300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.24299999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051637,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:31:35 2023","interfaces":[{"interface":"back","average":{"1min":0.26100000000000001,"5min":0.25700000000000001,"15min":0.251},"min":{"1min":0.19,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.35899999999999999,"5min":0.41599999999999998,"15min":0.41599999999999998},"last":0.221},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.27400000000000002,"15min":0.26700000000000002},"min":{"1min":0.20200000000000001,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.374,"5min":0.42899999999999999,"15min":0.42899999999999999},"last":0.24099999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:32:01.906+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:32:02.388+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:32:02.794 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:32:22.326 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":233,"stamp":"2023-03-25T04:32:22.153289+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002545"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:33.309748+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:29:33.309748+0000","last_peered":"2023-03-25T04:29:33.309748+0000","last_clean":"2023-03-25T04:29:33.309748+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:29:33.309748+0000","last_undegraded":"2023-03-25T04:29:33.309748+0000","last_fullsized":"2023-03-25T04:29:33.309748+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:33.309729+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:29:33.309729+0000","last_peered":"2023-03-25T04:29:33.309729+0000","last_clean":"2023-03-25T04:29:33.309729+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:29:33.309729+0000","last_undegraded":"2023-03-25T04:29:33.309729+0000","last_fullsized":"2023-03-25T04:29:33.309729+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:23.308607+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:29:23.308607+0000","last_peered":"2023-03-25T04:29:23.308607+0000","last_clean":"2023-03-25T04:29:23.308607+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:29:23.308607+0000","last_undegraded":"2023-03-25T04:29:23.308607+0000","last_fullsized":"2023-03-25T04:29:23.308607+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":28,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:20.554710+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:29:20.554710+0000","last_peered":"2023-03-25T04:29:20.554710+0000","last_clean":"2023-03-25T04:29:20.554710+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:29:20.554710+0000","last_undegraded":"2023-03-25T04:29:20.554710+0000","last_fullsized":"2023-03-25T04:29:20.554710+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309216+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:29:28.309216+0000","last_peered":"2023-03-25T04:29:28.309216+0000","last_clean":"2023-03-25T04:29:28.309216+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:29:28.309216+0000","last_undegraded":"2023-03-25T04:29:28.309216+0000","last_fullsized":"2023-03-25T04:29:28.309216+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309233+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:29:28.309233+0000","last_peered":"2023-03-25T04:29:28.309233+0000","last_clean":"2023-03-25T04:29:28.309233+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:29:28.309233+0000","last_undegraded":"2023-03-25T04:29:28.309233+0000","last_fullsized":"2023-03-25T04:29:28.309233+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117055,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:31:42 2023","interfaces":[{"interface":"back","average":{"1min":0.27500000000000002,"5min":0.28399999999999997,"15min":0.29399999999999998},"min":{"1min":0.19800000000000001,"5min":0.191,"15min":0.191},"max":{"1min":0.36899999999999999,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.32300000000000001},{"interface":"front","average":{"1min":0.27900000000000003,"5min":0.28499999999999998,"15min":0.29599999999999999},"min":{"1min":0.216,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":0.40799999999999997,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.27100000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051646,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:31:35 2023","interfaces":[{"interface":"back","average":{"1min":0.26100000000000001,"5min":0.25700000000000001,"15min":0.251},"min":{"1min":0.19,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.35899999999999999,"5min":0.41599999999999998,"15min":0.41599999999999998},"last":0.25600000000000001},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.27400000000000002,"15min":0.26700000000000002},"min":{"1min":0.20200000000000001,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.374,"5min":0.42899999999999999,"15min":0.42899999999999999},"last":0.29199999999999998}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:32:22.327 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean
, 2023-03-25T04:32:22.327 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}

Ignoring malparsed backtrace: 2023-03-25T04:32:39.836 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:32:39.849 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":241,"stamp":"2023-03-25T04:32:38.156312+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002346"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:33.309748+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:29:33.309748+0000","last_peered":"2023-03-25T04:29:33.309748+0000","last_clean":"2023-03-25T04:29:33.309748+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:29:33.309748+0000","last_undegraded":"2023-03-25T04:29:33.309748+0000","last_fullsized":"2023-03-25T04:29:33.309748+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:33.309729+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:29:33.309729+0000","last_peered":"2023-03-25T04:29:33.309729+0000","last_clean":"2023-03-25T04:29:33.309729+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:29:33.309729+0000","last_undegraded":"2023-03-25T04:29:33.309729+0000","last_fullsized":"2023-03-25T04:29:33.309729+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:23.308607+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:29:23.308607+0000","last_peered":"2023-03-25T04:29:23.308607+0000","last_clean":"2023-03-25T04:29:23.308607+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:29:23.308607+0000","last_undegraded":"2023-03-25T04:29:23.308607+0000","last_fullsized":"2023-03-25T04:29:23.308607+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":28,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:20.554710+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:29:20.554710+0000","last_peered":"2023-03-25T04:29:20.554710+0000","last_clean":"2023-03-25T04:29:20.554710+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:29:20.554710+0000","last_undegraded":"2023-03-25T04:29:20.554710+0000","last_fullsized":"2023-03-25T04:29:20.554710+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309216+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:29:28.309216+0000","last_peered":"2023-03-25T04:29:28.309216+0000","last_clean":"2023-03-25T04:29:28.309216+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:29:28.309216+0000","last_undegraded":"2023-03-25T04:29:28.309216+0000","last_fullsized":"2023-03-25T04:29:28.309216+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309233+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:29:28.309233+0000","last_peered":"2023-03-25T04:29:28.309233+0000","last_clean":"2023-03-25T04:29:28.309233+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:29:28.309233+0000","last_undegraded":"2023-03-25T04:29:28.309233+0000","last_fullsized":"2023-03-25T04:29:28.309233+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117058,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:31:42 2023","interfaces":[{"interface":"back","average":{"1min":0.27500000000000002,"5min":0.28399999999999997,"15min":0.29399999999999998},"min":{"1min":0.19800000000000001,"5min":0.191,"15min":0.191},"max":{"1min":0.36899999999999999,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.30599999999999999},{"interface":"front","average":{"1min":0.27900000000000003,"5min":0.28499999999999998,"15min":0.29599999999999999},"min":{"1min":0.216,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":0.40799999999999997,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.32500000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051649,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:31:35 2023","interfaces":[{"interface":"back","average":{"1min":0.26100000000000001,"5min":0.25700000000000001,"15min":0.251},"min":{"1min":0.19,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.35899999999999999,"5min":0.41599999999999998,"15min":0.41599999999999998},"last":0.24199999999999999},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.27400000000000002,"15min":0.26700000000000002},"min":{"1min":0.20200000000000001,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.374,"5min":0.42899999999999999,"15min":0.42899999999999999},"last":0.22600000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:32:54.944+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:32:55.469+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:32:55.927+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)

Ignoring malparsed backtrace: 2023-03-25T04:33:13.958 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean
, 2023-03-25T04:33:13.959 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}
, 2023-03-25T04:33:13.959 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json

Ignoring malparsed backtrace: 2023-03-25T04:33:31.199 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":267,"stamp":"2023-03-25T04:33:30.166660+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002269"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:33.309748+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:29:33.309748+0000","last_peered":"2023-03-25T04:29:33.309748+0000","last_clean":"2023-03-25T04:29:33.309748+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:29:33.309748+0000","last_undegraded":"2023-03-25T04:29:33.309748+0000","last_fullsized":"2023-03-25T04:29:33.309748+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:33.309729+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:29:33.309729+0000","last_peered":"2023-03-25T04:29:33.309729+0000","last_clean":"2023-03-25T04:29:33.309729+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:29:33.309729+0000","last_undegraded":"2023-03-25T04:29:33.309729+0000","last_fullsized":"2023-03-25T04:29:33.309729+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":28,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:31:23.320894+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:31:23.320894+0000","last_peered":"2023-03-25T04:31:23.320894+0000","last_clean":"2023-03-25T04:31:23.320894+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:31:23.320894+0000","last_undegraded":"2023-03-25T04:31:23.320894+0000","last_fullsized":"2023-03-25T04:31:23.320894+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":28,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:20.554710+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:29:20.554710+0000","last_peered":"2023-03-25T04:29:20.554710+0000","last_clean":"2023-03-25T04:29:20.554710+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:29:20.554710+0000","last_undegraded":"2023-03-25T04:29:20.554710+0000","last_fullsized":"2023-03-25T04:29:20.554710+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309216+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:29:28.309216+0000","last_peered":"2023-03-25T04:29:28.309216+0000","last_clean":"2023-03-25T04:29:28.309216+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:29:28.309216+0000","last_undegraded":"2023-03-25T04:29:28.309216+0000","last_fullsized":"2023-03-25T04:29:28.309216+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309233+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:29:28.309233+0000","last_peered":"2023-03-25T04:29:28.309233+0000","last_clean":"2023-03-25T04:29:28.309233+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:29:28.309233+0000","last_undegraded":"2023-03-25T04:29:28.309233+0000","last_fullsized":"2023-03-25T04:29:28.309233+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117068,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:32:46 2023","interfaces":[{"interface":"back","average":{"1min":0.27000000000000002,"5min":0.27900000000000003,"15min":0.29199999999999998},"min":{"1min":0.17899999999999999,"5min":0.17899999999999999,"15min":0.17899999999999999},"max":{"1min":0.35599999999999998,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.28799999999999998},{"interface":"front","average":{"1min":0.26800000000000002,"5min":0.27800000000000002,"15min":0.29399999999999998},"min":{"1min":0.19900000000000001,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":0.377,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.315}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051660,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:32:36 2023","interfaces":[{"interface":"back","average":{"1min":0.248,"5min":0.25700000000000001,"15min":0.251},"min":{"1min":0.17000000000000001,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"last":0.22700000000000001},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.27600000000000002,"15min":0.26800000000000002},"min":{"1min":0.192,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"last":0.20399999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:33:31.201 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:33:31.400+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)

Ignoring malparsed backtrace: 2023-03-25T04:33:47.531+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:33:47.951+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:33:48.505+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)

Ignoring malparsed backtrace: 2023-03-25T04:34:06.104 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json
, 2023-03-25T04:34:06.118+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:34:06.497 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:34:23.399 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:34:23.665+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:34:23.712 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:34:40.195+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:34:40.523+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:34:40.676 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace: 2023-03-25T04:35:03.381+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:35:04.139+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:35:04.391+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)

Ignoring malparsed backtrace: 2023-03-25T04:35:22.738 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean
, 2023-03-25T04:35:22.739 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}
, 2023-03-25T04:35:22.739 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json

Ignoring malparsed backtrace: 2023-03-25T04:35:41.393+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:35:42.352+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:35:42.376+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:35:57.384 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":340,"stamp":"2023-03-25T04:35:56.194992+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.001930"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":36,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:31:43.322894+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:31:43.322894+0000","last_peered":"2023-03-25T04:31:43.322894+0000","last_clean":"2023-03-25T04:31:43.322894+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:31:43.322894+0000","last_undegraded":"2023-03-25T04:31:43.322894+0000","last_fullsized":"2023-03-25T04:31:43.322894+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":28,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:31:43.322876+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:31:43.322876+0000","last_peered":"2023-03-25T04:31:43.322876+0000","last_clean":"2023-03-25T04:31:43.322876+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:31:43.322876+0000","last_undegraded":"2023-03-25T04:31:43.322876+0000","last_fullsized":"2023-03-25T04:31:43.322876+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:23.331998+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:33:23.331998+0000","last_peered":"2023-03-25T04:33:23.331998+0000","last_clean":"2023-03-25T04:33:23.331998+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:33:23.331998+0000","last_undegraded":"2023-03-25T04:33:23.331998+0000","last_fullsized":"2023-03-25T04:33:23.331998+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:30.576723+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:33:30.576723+0000","last_peered":"2023-03-25T04:33:30.576723+0000","last_clean":"2023-03-25T04:33:30.576723+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:33:30.576723+0000","last_undegraded":"2023-03-25T04:33:30.576723+0000","last_fullsized":"2023-03-25T04:33:30.576723+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":37,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:38.333447+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:33:38.333447+0000","last_peered":"2023-03-25T04:33:38.333447+0000","last_clean":"2023-03-25T04:33:38.333447+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:33:38.333447+0000","last_undegraded":"2023-03-25T04:33:38.333447+0000","last_fullsized":"2023-03-25T04:33:38.333447+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:38.333467+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:33:38.333467+0000","last_peered":"2023-03-25T04:33:38.333467+0000","last_clean":"2023-03-25T04:33:38.333467+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:33:38.333467+0000","last_undegraded":"2023-03-25T04:33:38.333467+0000","last_fullsized":"2023-03-25T04:33:38.333467+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117098,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:35:49 2023","interfaces":[{"interface":"back","average":{"1min":0.255,"5min":0.26000000000000001,"15min":0.28199999999999997},"min":{"1min":0.17899999999999999,"5min":0.16,"15min":0.16},"max":{"1min":0.36399999999999999,"5min":0.42599999999999999,"15min":0.45100000000000001},"last":0.26100000000000001},{"interface":"front","average":{"1min":0.26000000000000001,"5min":0.26700000000000002,"15min":0.28599999999999998},"min":{"1min":0.188,"5min":0.17999999999999999,"15min":0.17499999999999999},"max":{"1min":0.39100000000000001,"5min":0.45200000000000001,"15min":0.46300000000000002},"last":0.28399999999999997}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051689,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:35:42 2023","interfaces":[{"interface":"back","average":{"1min":0.29399999999999998,"5min":0.27200000000000002,"15min":0.25800000000000001},"min":{"1min":0.17299999999999999,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.41099999999999998,"5min":0.42699999999999999,"15min":0.42699999999999999},"last":0.20399999999999999},{"interface":"front","average":{"1min":0.30299999999999999,"5min":0.28399999999999997,"15min":0.27300000000000002},"min":{"1min":0.19700000000000001,"5min":0.192,"15min":0.16600000000000001},"max":{"1min":0.43099999999999999,"5min":0.45800000000000002,"15min":0.45800000000000002},"last":0.22}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:35:57.624+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)

Ignoring malparsed backtrace: 2023-03-25T04:36:17.226 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:36:17.239 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":350,"stamp":"2023-03-25T04:36:16.198866+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002148"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":37,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:53.335026+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:33:53.335026+0000","last_peered":"2023-03-25T04:33:53.335026+0000","last_clean":"2023-03-25T04:33:53.335026+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:33:53.335026+0000","last_undegraded":"2023-03-25T04:33:53.335026+0000","last_fullsized":"2023-03-25T04:33:53.335026+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:53.335009+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:33:53.335009+0000","last_peered":"2023-03-25T04:33:53.335009+0000","last_clean":"2023-03-25T04:33:53.335009+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:33:53.335009+0000","last_undegraded":"2023-03-25T04:33:53.335009+0000","last_fullsized":"2023-03-25T04:33:53.335009+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:23.331998+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:33:23.331998+0000","last_peered":"2023-03-25T04:33:23.331998+0000","last_clean":"2023-03-25T04:33:23.331998+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:33:23.331998+0000","last_undegraded":"2023-03-25T04:33:23.331998+0000","last_fullsized":"2023-03-25T04:33:23.331998+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:30.576723+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:33:30.576723+0000","last_peered":"2023-03-25T04:33:30.576723+0000","last_clean":"2023-03-25T04:33:30.576723+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:33:30.576723+0000","last_undegraded":"2023-03-25T04:33:30.576723+0000","last_fullsized":"2023-03-25T04:33:30.576723+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":37,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:38.333447+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:33:38.333447+0000","last_peered":"2023-03-25T04:33:38.333447+0000","last_clean":"2023-03-25T04:33:38.333447+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:33:38.333447+0000","last_undegraded":"2023-03-25T04:33:38.333447+0000","last_fullsized":"2023-03-25T04:33:38.333447+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:38.333467+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:33:38.333467+0000","last_peered":"2023-03-25T04:33:38.333467+0000","last_clean":"2023-03-25T04:33:38.333467+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:33:38.333467+0000","last_undegraded":"2023-03-25T04:33:38.333467+0000","last_fullsized":"2023-03-25T04:33:38.333467+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117102,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:35:49 2023","interfaces":[{"interface":"back","average":{"1min":0.255,"5min":0.26000000000000001,"15min":0.28199999999999997},"min":{"1min":0.17899999999999999,"5min":0.16,"15min":0.16},"max":{"1min":0.36399999999999999,"5min":0.42599999999999999,"15min":0.45100000000000001},"last":0.28399999999999997},{"interface":"front","average":{"1min":0.26000000000000001,"5min":0.26700000000000002,"15min":0.28599999999999998},"min":{"1min":0.188,"5min":0.17999999999999999,"15min":0.17499999999999999},"max":{"1min":0.39100000000000001,"5min":0.45200000000000001,"15min":0.46300000000000002},"last":0.30299999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051693,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:35:42 2023","interfaces":[{"interface":"back","average":{"1min":0.29399999999999998,"5min":0.27200000000000002,"15min":0.25800000000000001},"min":{"1min":0.17299999999999999,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.41099999999999998,"5min":0.42699999999999999,"15min":0.42699999999999999},"last":0.35099999999999998},{"interface":"front","average":{"1min":0.30299999999999999,"5min":0.28399999999999997,"15min":0.27300000000000002},"min":{"1min":0.19700000000000001,"5min":0.192,"15min":0.16600000000000001},"max":{"1min":0.43099999999999999,"5min":0.45800000000000002,"15min":0.45800000000000002},"last":0.32900000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:36:34.556+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:36:34.717+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:36:35.583+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)

Ignoring malparsed backtrace: 2023-03-25T04:36:52.680 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json
, 2023-03-25T04:36:52.701+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:36:53.067 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:37:09.963 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:37:10.273 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all

Ignoring malparsed backtrace: 2023-03-25T04:37:26.856+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:37:27.288 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:37:27.538+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:37:41.903 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":392,"stamp":"2023-03-25T04:37:40.213373+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002012"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":37,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:53.335026+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:33:53.335026+0000","last_peered":"2023-03-25T04:33:53.335026+0000","last_clean":"2023-03-25T04:33:53.335026+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:33:53.335026+0000","last_undegraded":"2023-03-25T04:33:53.335026+0000","last_fullsized":"2023-03-25T04:33:53.335026+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:53.335009+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:33:53.335009+0000","last_peered":"2023-03-25T04:33:53.335009+0000","last_clean":"2023-03-25T04:33:53.335009+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:33:53.335009+0000","last_undegraded":"2023-03-25T04:33:53.335009+0000","last_fullsized":"2023-03-25T04:33:53.335009+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:23.343585+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:35:23.343585+0000","last_peered":"2023-03-25T04:35:23.343585+0000","last_clean":"2023-03-25T04:35:23.343585+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:35:23.343585+0000","last_undegraded":"2023-03-25T04:35:23.343585+0000","last_fullsized":"2023-03-25T04:35:23.343585+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:30.576723+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:33:30.576723+0000","last_peered":"2023-03-25T04:33:30.576723+0000","last_clean":"2023-03-25T04:33:30.576723+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:33:30.576723+0000","last_undegraded":"2023-03-25T04:33:30.576723+0000","last_fullsized":"2023-03-25T04:33:30.576723+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":37,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:38.333447+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:33:38.333447+0000","last_peered":"2023-03-25T04:33:38.333447+0000","last_clean":"2023-03-25T04:33:38.333447+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:33:38.333447+0000","last_undegraded":"2023-03-25T04:33:38.333447+0000","last_fullsized":"2023-03-25T04:33:38.333447+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:38.333467+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:33:38.333467+0000","last_peered":"2023-03-25T04:33:38.333467+0000","last_clean":"2023-03-25T04:33:38.333467+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:33:38.333467+0000","last_undegraded":"2023-03-25T04:33:38.333467+0000","last_fullsized":"2023-03-25T04:33:38.333467+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117119,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:36:50 2023","interfaces":[{"interface":"back","average":{"1min":0.254,"5min":0.25600000000000001,"15min":0.27900000000000003},"min":{"1min":0.20499999999999999,"5min":0.16,"15min":0.16},"max":{"1min":0.371,"5min":0.42599999999999999,"15min":0.45100000000000001},"last":0.23000000000000001},{"interface":"front","average":{"1min":0.26800000000000002,"5min":0.26500000000000001,"15min":0.28399999999999997},"min":{"1min":0.19700000000000001,"5min":0.17999999999999999,"15min":0.17499999999999999},"max":{"1min":0.34699999999999998,"5min":0.45200000000000001,"15min":0.46300000000000002},"last":0.20899999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051710,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:36:44 2023","interfaces":[{"interface":"back","average":{"1min":0.28399999999999997,"5min":0.27700000000000002,"15min":0.26100000000000001},"min":{"1min":0.192,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.40899999999999997,"5min":0.42699999999999999,"15min":0.42699999999999999},"last":0.39700000000000002},{"interface":"front","average":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.27500000000000002},"min":{"1min":0.182,"5min":0.182,"15min":0.16600000000000001},"max":{"1min":0.439,"5min":0.45800000000000002,"15min":0.45800000000000002},"last":0.439}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:37:42.400+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)

Ignoring malparsed backtrace: 2023-03-25T04:38:04.766+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:38:04.922 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:38:05.246 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:38:19.605 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":411,"stamp":"2023-03-25T04:38:18.220966+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002368"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":38,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:36:03.347070+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:36:03.347070+0000","last_peered":"2023-03-25T04:36:03.347070+0000","last_clean":"2023-03-25T04:36:03.347070+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:36:03.347070+0000","last_undegraded":"2023-03-25T04:36:03.347070+0000","last_fullsized":"2023-03-25T04:36:03.347070+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:36:03.347025+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:36:03.347025+0000","last_peered":"2023-03-25T04:36:03.347025+0000","last_clean":"2023-03-25T04:36:03.347025+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:36:03.347025+0000","last_undegraded":"2023-03-25T04:36:03.347025+0000","last_fullsized":"2023-03-25T04:36:03.347025+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:23.343585+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:35:23.343585+0000","last_peered":"2023-03-25T04:35:23.343585+0000","last_clean":"2023-03-25T04:35:23.343585+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:35:23.343585+0000","last_undegraded":"2023-03-25T04:35:23.343585+0000","last_fullsized":"2023-03-25T04:35:23.343585+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:35.587025+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:35:35.587025+0000","last_peered":"2023-03-25T04:35:35.587025+0000","last_clean":"2023-03-25T04:35:35.587025+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:35:35.587025+0000","last_undegraded":"2023-03-25T04:35:35.587025+0000","last_fullsized":"2023-03-25T04:35:35.587025+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":38,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:43.345308+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:35:43.345308+0000","last_peered":"2023-03-25T04:35:43.345308+0000","last_clean":"2023-03-25T04:35:43.345308+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:35:43.345308+0000","last_undegraded":"2023-03-25T04:35:43.345308+0000","last_fullsized":"2023-03-25T04:35:43.345308+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:43.345324+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:35:43.345324+0000","last_peered":"2023-03-25T04:35:43.345324+0000","last_clean":"2023-03-25T04:35:43.345324+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:35:43.345324+0000","last_undegraded":"2023-03-25T04:35:43.345324+0000","last_fullsized":"2023-03-25T04:35:43.345324+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117127,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:37:53 2023","interfaces":[{"interface":"back","average":{"1min":0.25800000000000001,"5min":0.253,"15min":0.27700000000000002},"min":{"1min":0.184,"5min":0.16,"15min":0.16},"max":{"1min":0.44600000000000001,"5min":0.44600000000000001,"15min":0.45100000000000001},"last":0.35199999999999998},{"interface":"front","average":{"1min":0.26900000000000002,"5min":0.26500000000000001,"15min":0.28199999999999997},"min":{"1min":0.192,"5min":0.17999999999999999,"15min":0.17499999999999999},"max":{"1min":0.46400000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"last":0.33200000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051717,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:37:44 2023","interfaces":[{"interface":"back","average":{"1min":0.27300000000000002,"5min":0.28199999999999997,"15min":0.26200000000000001},"min":{"1min":0.17899999999999999,"5min":0.17299999999999999,"15min":0.17000000000000001},"max":{"1min":0.41799999999999998,"5min":0.42099999999999999,"15min":0.42699999999999999},"last":0.28699999999999998},{"interface":"front","average":{"1min":0.29399999999999998,"5min":0.29099999999999998,"15min":0.27700000000000002},"min":{"1min":0.19900000000000001,"5min":0.182,"15min":0.16600000000000001},"max":{"1min":0.441,"5min":0.441,"15min":0.45800000000000002},"last":0.26300000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:38:19.718+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:38:20.315+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)

Ignoring malparsed backtrace: 2023-03-25T04:38:44.283 INFO:teuthology.orchestra.run.smithi070.stdout:
, 2023-03-25T04:38:44.283 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":781,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":16,"modified":"2023-03-25T04:38:40.225321+0000","services":{}},"progress_events":{}}
, 2023-03-25T04:38:44.296 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now

Ignoring malparsed backtrace: 2023-03-25T04:39:02.477+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:39:02.578+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:39:03.524+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:39:20.359 INFO:teuthology.orchestra.run.smithi070.stdout:
, 2023-03-25T04:39:20.359 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":818,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":16,"modified":"2023-03-25T04:38:40.225321+0000","services":{}},"progress_events":{}}
, 2023-03-25T04:39:20.372 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now

Ignoring malparsed backtrace: 2023-03-25T04:39:37.527 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:39:37.541 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":450,"stamp":"2023-03-25T04:39:36.235332+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002086"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":38,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:36:03.347070+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:36:03.347070+0000","last_peered":"2023-03-25T04:36:03.347070+0000","last_clean":"2023-03-25T04:36:03.347070+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:36:03.347070+0000","last_undegraded":"2023-03-25T04:36:03.347070+0000","last_fullsized":"2023-03-25T04:36:03.347070+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:36:03.347025+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:36:03.347025+0000","last_peered":"2023-03-25T04:36:03.347025+0000","last_clean":"2023-03-25T04:36:03.347025+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:36:03.347025+0000","last_undegraded":"2023-03-25T04:36:03.347025+0000","last_fullsized":"2023-03-25T04:36:03.347025+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:23.353778+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:37:23.353778+0000","last_peered":"2023-03-25T04:37:23.353778+0000","last_clean":"2023-03-25T04:37:23.353778+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:37:23.353778+0000","last_undegraded":"2023-03-25T04:37:23.353778+0000","last_fullsized":"2023-03-25T04:37:23.353778+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:35.587025+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:35:35.587025+0000","last_peered":"2023-03-25T04:35:35.587025+0000","last_clean":"2023-03-25T04:35:35.587025+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:35:35.587025+0000","last_undegraded":"2023-03-25T04:35:35.587025+0000","last_fullsized":"2023-03-25T04:35:35.587025+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":38,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:43.345308+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:35:43.345308+0000","last_peered":"2023-03-25T04:35:43.345308+0000","last_clean":"2023-03-25T04:35:43.345308+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:35:43.345308+0000","last_undegraded":"2023-03-25T04:35:43.345308+0000","last_fullsized":"2023-03-25T04:35:43.345308+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:43.345324+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:35:43.345324+0000","last_peered":"2023-03-25T04:35:43.345324+0000","last_clean":"2023-03-25T04:35:43.345324+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:35:43.345324+0000","last_undegraded":"2023-03-25T04:35:43.345324+0000","last_fullsized":"2023-03-25T04:35:43.345324+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117143,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:38:54 2023","interfaces":[{"interface":"back","average":{"1min":0.25800000000000001,"5min":0.25700000000000001,"15min":0.27400000000000002},"min":{"1min":0.192,"5min":0.17399999999999999,"15min":0.16},"max":{"1min":0.39400000000000002,"5min":0.44600000000000001,"15min":0.45100000000000001},"last":0.26500000000000001},{"interface":"front","average":{"1min":0.25900000000000001,"5min":0.26700000000000002,"15min":0.27900000000000003},"min":{"1min":0.19500000000000001,"5min":0.188,"15min":0.17499999999999999},"max":{"1min":0.41499999999999998,"5min":0.46400000000000002,"15min":0.46400000000000002},"last":0.29099999999999998}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051733,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:38:45 2023","interfaces":[{"interface":"back","average":{"1min":0.27300000000000002,"5min":0.28100000000000003,"15min":0.26400000000000001},"min":{"1min":0.19400000000000001,"5min":0.17299999999999999,"15min":0.17000000000000001},"max":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"last":0.27300000000000002},{"interface":"front","average":{"1min":0.28799999999999998,"5min":0.29099999999999998,"15min":0.27800000000000002},"min":{"1min":0.19800000000000001,"5min":0.182,"15min":0.16600000000000001},"max":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"last":0.308}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:39:54.416+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:39:54.455 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:39:54.780 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:40:09.288+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:40:09.389+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:40:10.267+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)

Ignoring malparsed backtrace: 2023-03-25T04:40:28.984 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":476,"stamp":"2023-03-25T04:40:28.245911+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002693"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":39,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:38:13.358968+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:38:13.358968+0000","last_peered":"2023-03-25T04:38:13.358968+0000","last_clean":"2023-03-25T04:38:13.358968+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:38:13.358968+0000","last_undegraded":"2023-03-25T04:38:13.358968+0000","last_fullsized":"2023-03-25T04:38:13.358968+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:38:13.358922+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:38:13.358922+0000","last_peered":"2023-03-25T04:38:13.358922+0000","last_clean":"2023-03-25T04:38:13.358922+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:38:13.358922+0000","last_undegraded":"2023-03-25T04:38:13.358922+0000","last_fullsized":"2023-03-25T04:38:13.358922+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:23.353778+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:37:23.353778+0000","last_peered":"2023-03-25T04:37:23.353778+0000","last_clean":"2023-03-25T04:37:23.353778+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:37:23.353778+0000","last_undegraded":"2023-03-25T04:37:23.353778+0000","last_fullsized":"2023-03-25T04:37:23.353778+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:40.598579+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:37:40.598579+0000","last_peered":"2023-03-25T04:37:40.598579+0000","last_clean":"2023-03-25T04:37:40.598579+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:37:40.598579+0000","last_undegraded":"2023-03-25T04:37:40.598579+0000","last_fullsized":"2023-03-25T04:37:40.598579+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":39,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:48.356344+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:37:48.356344+0000","last_peered":"2023-03-25T04:37:48.356344+0000","last_clean":"2023-03-25T04:37:48.356344+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:37:48.356344+0000","last_undegraded":"2023-03-25T04:37:48.356344+0000","last_fullsized":"2023-03-25T04:37:48.356344+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:48.356365+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:37:48.356365+0000","last_peered":"2023-03-25T04:37:48.356365+0000","last_clean":"2023-03-25T04:37:48.356365+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:37:48.356365+0000","last_undegraded":"2023-03-25T04:37:48.356365+0000","last_fullsized":"2023-03-25T04:37:48.356365+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117153,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:39:54 2023","interfaces":[{"interface":"back","average":{"1min":0.25900000000000001,"5min":0.25600000000000001,"15min":0.27100000000000002},"min":{"1min":0.188,"5min":0.17899999999999999,"15min":0.16},"max":{"1min":0.41999999999999998,"5min":0.44600000000000001,"15min":0.45100000000000001},"last":0.255},{"interface":"front","average":{"1min":0.27500000000000002,"5min":0.26600000000000001,"15min":0.27700000000000002},"min":{"1min":0.17699999999999999,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.45600000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"last":0.26800000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051743,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:39:47 2023","interfaces":[{"interface":"back","average":{"1min":0.26300000000000001,"5min":0.27700000000000002,"15min":0.26500000000000001},"min":{"1min":0.187,"5min":0.17299999999999999,"15min":0.17000000000000001},"max":{"1min":0.433,"5min":0.433,"15min":0.433},"last":0.36599999999999999},{"interface":"front","average":{"1min":0.27100000000000002,"5min":0.28799999999999998,"15min":0.27900000000000003},"min":{"1min":0.20100000000000001,"5min":0.182,"15min":0.16600000000000001},"max":{"1min":0.48599999999999999,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.34999999999999998}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:40:28.986 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean
, 2023-03-25T04:40:28.986 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}

Ignoring malparsed backtrace: 2023-03-25T04:40:47.229+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:40:47.503+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:40:48.251+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)

Ignoring malparsed backtrace: 2023-03-25T04:41:03.267 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:41:03.578 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all

Ignoring malparsed backtrace: 2023-03-25T04:41:21.347 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":502,"stamp":"2023-03-25T04:41:20.256204+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002342"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":39,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:38:13.358968+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:38:13.358968+0000","last_peered":"2023-03-25T04:38:13.358968+0000","last_clean":"2023-03-25T04:38:13.358968+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:38:13.358968+0000","last_undegraded":"2023-03-25T04:38:13.358968+0000","last_fullsized":"2023-03-25T04:38:13.358968+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:38:13.358922+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:38:13.358922+0000","last_peered":"2023-03-25T04:38:13.358922+0000","last_clean":"2023-03-25T04:38:13.358922+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:38:13.358922+0000","last_undegraded":"2023-03-25T04:38:13.358922+0000","last_fullsized":"2023-03-25T04:38:13.358922+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:23.353778+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:37:23.353778+0000","last_peered":"2023-03-25T04:37:23.353778+0000","last_clean":"2023-03-25T04:37:23.353778+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:37:23.353778+0000","last_undegraded":"2023-03-25T04:37:23.353778+0000","last_fullsized":"2023-03-25T04:37:23.353778+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:40.598579+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:37:40.598579+0000","last_peered":"2023-03-25T04:37:40.598579+0000","last_clean":"2023-03-25T04:37:40.598579+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:37:40.598579+0000","last_undegraded":"2023-03-25T04:37:40.598579+0000","last_fullsized":"2023-03-25T04:37:40.598579+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":39,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:48.356344+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:37:48.356344+0000","last_peered":"2023-03-25T04:37:48.356344+0000","last_clean":"2023-03-25T04:37:48.356344+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:37:48.356344+0000","last_undegraded":"2023-03-25T04:37:48.356344+0000","last_fullsized":"2023-03-25T04:37:48.356344+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:48.356365+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:37:48.356365+0000","last_peered":"2023-03-25T04:37:48.356365+0000","last_clean":"2023-03-25T04:37:48.356365+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:37:48.356365+0000","last_undegraded":"2023-03-25T04:37:48.356365+0000","last_fullsized":"2023-03-25T04:37:48.356365+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117163,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:40:56 2023","interfaces":[{"interface":"back","average":{"1min":0.28799999999999998,"5min":0.26300000000000001,"15min":0.27100000000000002},"min":{"1min":0.188,"5min":0.184,"15min":0.16},"max":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.17499999999999999},{"interface":"front","average":{"1min":0.29599999999999999,"5min":0.27300000000000002,"15min":0.27700000000000002},"min":{"1min":0.20599999999999999,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.43099999999999999,"5min":0.46400000000000002,"15min":0.46400000000000002},"last":0.20699999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051754,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:40:52 2023","interfaces":[{"interface":"back","average":{"1min":0.28599999999999998,"5min":0.27500000000000002,"15min":0.26700000000000002},"min":{"1min":0.17899999999999999,"5min":0.17899999999999999,"15min":0.17000000000000001},"max":{"1min":0.44,"5min":0.44,"15min":0.44},"last":0.23699999999999999},{"interface":"front","average":{"1min":0.29799999999999999,"5min":0.28699999999999998,"15min":0.28100000000000003},"min":{"1min":0.20300000000000001,"5min":0.182,"15min":0.16600000000000001},"max":{"1min":0.47599999999999998,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.28000000000000003}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:41:21.349 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean
, 2023-03-25T04:41:21.349 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}

Ignoring malparsed backtrace: 2023-03-25T04:41:39.867+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:41:40.383+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:41:40.905+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:41:58.219 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json
, 2023-03-25T04:41:58.319+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:41:58.644 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:42:18.193 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":530,"stamp":"2023-03-25T04:42:16.266659+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002356"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":39,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:38:13.358968+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:38:13.358968+0000","last_peered":"2023-03-25T04:38:13.358968+0000","last_clean":"2023-03-25T04:38:13.358968+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:38:13.358968+0000","last_undegraded":"2023-03-25T04:38:13.358968+0000","last_fullsized":"2023-03-25T04:38:13.358968+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:38:13.358922+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:38:13.358922+0000","last_peered":"2023-03-25T04:38:13.358922+0000","last_clean":"2023-03-25T04:38:13.358922+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:38:13.358922+0000","last_undegraded":"2023-03-25T04:38:13.358922+0000","last_fullsized":"2023-03-25T04:38:13.358922+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:23.364593+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:39:23.364593+0000","last_peered":"2023-03-25T04:39:23.364593+0000","last_clean":"2023-03-25T04:39:23.364593+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:39:23.364593+0000","last_undegraded":"2023-03-25T04:39:23.364593+0000","last_fullsized":"2023-03-25T04:39:23.364593+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:45.608475+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:39:45.608475+0000","last_peered":"2023-03-25T04:39:45.608475+0000","last_clean":"2023-03-25T04:39:45.608475+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:39:45.608475+0000","last_undegraded":"2023-03-25T04:39:45.608475+0000","last_fullsized":"2023-03-25T04:39:45.608475+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":40,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:53.367398+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:39:53.367398+0000","last_peered":"2023-03-25T04:39:53.367398+0000","last_clean":"2023-03-25T04:39:53.367398+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:39:53.367398+0000","last_undegraded":"2023-03-25T04:39:53.367398+0000","last_fullsized":"2023-03-25T04:39:53.367398+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:53.367424+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:39:53.367424+0000","last_peered":"2023-03-25T04:39:53.367424+0000","last_clean":"2023-03-25T04:39:53.367424+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:39:53.367424+0000","last_undegraded":"2023-03-25T04:39:53.367424+0000","last_fullsized":"2023-03-25T04:39:53.367424+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117175,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:41:58 2023","interfaces":[{"interface":"back","average":{"1min":0.25800000000000001,"5min":0.26400000000000001,"15min":0.26800000000000002},"min":{"1min":0.17499999999999999,"5min":0.17499999999999999,"15min":0.16},"max":{"1min":0.39100000000000001,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.22900000000000001},{"interface":"front","average":{"1min":0.26300000000000001,"5min":0.27200000000000002,"15min":0.27400000000000002},"min":{"1min":0.20599999999999999,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.35699999999999998,"5min":0.46400000000000002,"15min":0.46400000000000002},"last":0.26400000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051765,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:41:52 2023","interfaces":[{"interface":"back","average":{"1min":0.26500000000000001,"5min":0.27200000000000002,"15min":0.26800000000000002},"min":{"1min":0.17000000000000001,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.39200000000000002,"5min":0.44,"15min":0.44},"last":0.23899999999999999},{"interface":"front","average":{"1min":0.28100000000000003,"5min":0.28599999999999998,"15min":0.28199999999999997},"min":{"1min":0.19400000000000001,"5min":0.19400000000000001,"15min":0.16600000000000001},"max":{"1min":0.41399999999999998,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.254}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:42:18.297+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:42:18.958+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:42:35.512 INFO:teuthology.orchestra.run.smithi070.stdout:
, 2023-03-25T04:42:35.512 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":1013,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":17,"modified":"2023-03-25T04:41:54.262744+0000","services":{}},"progress_events":{}}
, 2023-03-25T04:42:35.525 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:42:52.741 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":548,"stamp":"2023-03-25T04:42:52.273601+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002221"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":40,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:40:23.370056+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:40:23.370056+0000","last_peered":"2023-03-25T04:40:23.370056+0000","last_clean":"2023-03-25T04:40:23.370056+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:40:23.370056+0000","last_undegraded":"2023-03-25T04:40:23.370056+0000","last_fullsized":"2023-03-25T04:40:23.370056+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:40:23.370039+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:40:23.370039+0000","last_peered":"2023-03-25T04:40:23.370039+0000","last_clean":"2023-03-25T04:40:23.370039+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:40:23.370039+0000","last_undegraded":"2023-03-25T04:40:23.370039+0000","last_fullsized":"2023-03-25T04:40:23.370039+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:23.364593+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:39:23.364593+0000","last_peered":"2023-03-25T04:39:23.364593+0000","last_clean":"2023-03-25T04:39:23.364593+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:39:23.364593+0000","last_undegraded":"2023-03-25T04:39:23.364593+0000","last_fullsized":"2023-03-25T04:39:23.364593+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:45.608475+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:39:45.608475+0000","last_peered":"2023-03-25T04:39:45.608475+0000","last_clean":"2023-03-25T04:39:45.608475+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:39:45.608475+0000","last_undegraded":"2023-03-25T04:39:45.608475+0000","last_fullsized":"2023-03-25T04:39:45.608475+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":40,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:53.367398+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:39:53.367398+0000","last_peered":"2023-03-25T04:39:53.367398+0000","last_clean":"2023-03-25T04:39:53.367398+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:39:53.367398+0000","last_undegraded":"2023-03-25T04:39:53.367398+0000","last_fullsized":"2023-03-25T04:39:53.367398+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:53.367424+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:39:53.367424+0000","last_peered":"2023-03-25T04:39:53.367424+0000","last_clean":"2023-03-25T04:39:53.367424+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:39:53.367424+0000","last_undegraded":"2023-03-25T04:39:53.367424+0000","last_fullsized":"2023-03-25T04:39:53.367424+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117182,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:41:58 2023","interfaces":[{"interface":"back","average":{"1min":0.25800000000000001,"5min":0.26400000000000001,"15min":0.26800000000000002},"min":{"1min":0.17499999999999999,"5min":0.17499999999999999,"15min":0.16},"max":{"1min":0.39100000000000001,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.19600000000000001},{"interface":"front","average":{"1min":0.26300000000000001,"5min":0.27200000000000002,"15min":0.27400000000000002},"min":{"1min":0.20599999999999999,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.35699999999999998,"5min":0.46400000000000002,"15min":0.46400000000000002},"last":0.219}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051773,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:41:52 2023","interfaces":[{"interface":"back","average":{"1min":0.26500000000000001,"5min":0.27200000000000002,"15min":0.26800000000000002},"min":{"1min":0.17000000000000001,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.39200000000000002,"5min":0.44,"15min":0.44},"last":0.19},{"interface":"front","average":{"1min":0.28100000000000003,"5min":0.28599999999999998,"15min":0.28199999999999997},"min":{"1min":0.19400000000000001,"5min":0.19400000000000001,"15min":0.16600000000000001},"max":{"1min":0.41399999999999998,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.17599999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:42:52.742 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean

Ignoring malparsed backtrace: 2023-03-25T04:43:09.973 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:43:09.986 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":556,"stamp":"2023-03-25T04:43:08.276556+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002359"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":40,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:40:23.370056+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:40:23.370056+0000","last_peered":"2023-03-25T04:40:23.370056+0000","last_clean":"2023-03-25T04:40:23.370056+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:40:23.370056+0000","last_undegraded":"2023-03-25T04:40:23.370056+0000","last_fullsized":"2023-03-25T04:40:23.370056+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:40:23.370039+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:40:23.370039+0000","last_peered":"2023-03-25T04:40:23.370039+0000","last_clean":"2023-03-25T04:40:23.370039+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:40:23.370039+0000","last_undegraded":"2023-03-25T04:40:23.370039+0000","last_fullsized":"2023-03-25T04:40:23.370039+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:23.364593+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:39:23.364593+0000","last_peered":"2023-03-25T04:39:23.364593+0000","last_clean":"2023-03-25T04:39:23.364593+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:39:23.364593+0000","last_undegraded":"2023-03-25T04:39:23.364593+0000","last_fullsized":"2023-03-25T04:39:23.364593+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:45.608475+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:39:45.608475+0000","last_peered":"2023-03-25T04:39:45.608475+0000","last_clean":"2023-03-25T04:39:45.608475+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:39:45.608475+0000","last_undegraded":"2023-03-25T04:39:45.608475+0000","last_fullsized":"2023-03-25T04:39:45.608475+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":40,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:53.367398+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:39:53.367398+0000","last_peered":"2023-03-25T04:39:53.367398+0000","last_clean":"2023-03-25T04:39:53.367398+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:39:53.367398+0000","last_undegraded":"2023-03-25T04:39:53.367398+0000","last_fullsized":"2023-03-25T04:39:53.367398+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:53.367424+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:39:53.367424+0000","last_peered":"2023-03-25T04:39:53.367424+0000","last_clean":"2023-03-25T04:39:53.367424+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:39:53.367424+0000","last_undegraded":"2023-03-25T04:39:53.367424+0000","last_fullsized":"2023-03-25T04:39:53.367424+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117185,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:43:00 2023","interfaces":[{"interface":"back","average":{"1min":0.27400000000000002,"5min":0.26700000000000002,"15min":0.26600000000000001},"min":{"1min":0.19600000000000001,"5min":0.17499999999999999,"15min":0.16},"max":{"1min":0.42999999999999999,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.27800000000000002},{"interface":"front","average":{"1min":0.28199999999999997,"5min":0.27500000000000002,"15min":0.27300000000000002},"min":{"1min":0.185,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.47199999999999998,"5min":0.47199999999999998,"15min":0.47199999999999998},"last":0.247}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051776,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:42:54 2023","interfaces":[{"interface":"back","average":{"1min":0.26300000000000001,"5min":0.27000000000000002,"15min":0.26900000000000002},"min":{"1min":0.17399999999999999,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.438,"5min":0.44,"15min":0.44},"last":0.20100000000000001},{"interface":"front","average":{"1min":0.27400000000000002,"5min":0.28199999999999997,"15min":0.28299999999999997},"min":{"1min":0.17599999999999999,"5min":0.17599999999999999,"15min":0.16600000000000001},"max":{"1min":0.41599999999999998,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.23499999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:43:25.042+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:43:25.411+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:43:26.013+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:43:47.937+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:43:48.534+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:43:48.889+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:44:09.318 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:44:09.331 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":586,"stamp":"2023-03-25T04:44:08.288933+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002697"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":40,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:40:23.370056+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:40:23.370056+0000","last_peered":"2023-03-25T04:40:23.370056+0000","last_clean":"2023-03-25T04:40:23.370056+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:40:23.370056+0000","last_undegraded":"2023-03-25T04:40:23.370056+0000","last_fullsized":"2023-03-25T04:40:23.370056+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:40:23.370039+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:40:23.370039+0000","last_peered":"2023-03-25T04:40:23.370039+0000","last_clean":"2023-03-25T04:40:23.370039+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:40:23.370039+0000","last_undegraded":"2023-03-25T04:40:23.370039+0000","last_fullsized":"2023-03-25T04:40:23.370039+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:23.375106+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:41:23.375106+0000","last_peered":"2023-03-25T04:41:23.375106+0000","last_clean":"2023-03-25T04:41:23.375106+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:41:23.375106+0000","last_undegraded":"2023-03-25T04:41:23.375106+0000","last_fullsized":"2023-03-25T04:41:23.375106+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:50.618930+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:41:50.618930+0000","last_peered":"2023-03-25T04:41:50.618930+0000","last_clean":"2023-03-25T04:41:50.618930+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:41:50.618930+0000","last_undegraded":"2023-03-25T04:41:50.618930+0000","last_fullsized":"2023-03-25T04:41:50.618930+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":41,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:58.377821+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:41:58.377821+0000","last_peered":"2023-03-25T04:41:58.377821+0000","last_clean":"2023-03-25T04:41:58.377821+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:41:58.377821+0000","last_undegraded":"2023-03-25T04:41:58.377821+0000","last_fullsized":"2023-03-25T04:41:58.377821+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:58.377836+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:41:58.377836+0000","last_peered":"2023-03-25T04:41:58.377836+0000","last_clean":"2023-03-25T04:41:58.377836+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:41:58.377836+0000","last_undegraded":"2023-03-25T04:41:58.377836+0000","last_fullsized":"2023-03-25T04:41:58.377836+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117197,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:44:01 2023","interfaces":[{"interface":"back","average":{"1min":0.27200000000000002,"5min":0.27000000000000002,"15min":0.26400000000000001},"min":{"1min":0.20399999999999999,"5min":0.17499999999999999,"15min":0.16},"max":{"1min":0.38400000000000001,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.245},{"interface":"front","average":{"1min":0.27600000000000002,"5min":0.27800000000000002,"15min":0.27100000000000002},"min":{"1min":0.23100000000000001,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.40400000000000003,"5min":0.47199999999999998,"15min":0.47199999999999998},"last":0.27500000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051788,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:43:55 2023","interfaces":[{"interface":"back","average":{"1min":0.26100000000000001,"5min":0.26700000000000002,"15min":0.27000000000000002},"min":{"1min":0.17799999999999999,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.374,"5min":0.44,"15min":0.44},"last":0.30299999999999999},{"interface":"front","average":{"1min":0.28299999999999997,"5min":0.28100000000000003,"15min":0.28399999999999997},"min":{"1min":0.20899999999999999,"5min":0.17599999999999999,"15min":0.16600000000000001},"max":{"1min":0.40100000000000002,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.27800000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:44:28.923 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":1126,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":19,"modified":"2023-03-25T04:43:20.278959+0000","services":{}},"progress_events":{}}
, 2023-03-25T04:44:28.936 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now
, 2023-03-25T04:44:28.937 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace: 2023-03-25T04:44:48.786 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:44:48.800 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":606,"stamp":"2023-03-25T04:44:48.296805+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002182"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":41,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:42:34.382196+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:42:34.382196+0000","last_peered":"2023-03-25T04:42:34.382196+0000","last_clean":"2023-03-25T04:42:34.382196+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:42:34.382196+0000","last_undegraded":"2023-03-25T04:42:34.382196+0000","last_fullsized":"2023-03-25T04:42:34.382196+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:42:29.381744+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:42:29.381744+0000","last_peered":"2023-03-25T04:42:29.381744+0000","last_clean":"2023-03-25T04:42:29.381744+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:42:29.381744+0000","last_undegraded":"2023-03-25T04:42:29.381744+0000","last_fullsized":"2023-03-25T04:42:29.381744+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:23.375106+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:41:23.375106+0000","last_peered":"2023-03-25T04:41:23.375106+0000","last_clean":"2023-03-25T04:41:23.375106+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:41:23.375106+0000","last_undegraded":"2023-03-25T04:41:23.375106+0000","last_fullsized":"2023-03-25T04:41:23.375106+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:50.618930+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:41:50.618930+0000","last_peered":"2023-03-25T04:41:50.618930+0000","last_clean":"2023-03-25T04:41:50.618930+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:41:50.618930+0000","last_undegraded":"2023-03-25T04:41:50.618930+0000","last_fullsized":"2023-03-25T04:41:50.618930+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":41,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:58.377821+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:41:58.377821+0000","last_peered":"2023-03-25T04:41:58.377821+0000","last_clean":"2023-03-25T04:41:58.377821+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:41:58.377821+0000","last_undegraded":"2023-03-25T04:41:58.377821+0000","last_fullsized":"2023-03-25T04:41:58.377821+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:58.377836+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:41:58.377836+0000","last_peered":"2023-03-25T04:41:58.377836+0000","last_clean":"2023-03-25T04:41:58.377836+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:41:58.377836+0000","last_undegraded":"2023-03-25T04:41:58.377836+0000","last_fullsized":"2023-03-25T04:41:58.377836+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117206,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:44:01 2023","interfaces":[{"interface":"back","average":{"1min":0.27200000000000002,"5min":0.27000000000000002,"15min":0.26400000000000001},"min":{"1min":0.20399999999999999,"5min":0.17499999999999999,"15min":0.16},"max":{"1min":0.38400000000000001,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.216},{"interface":"front","average":{"1min":0.27600000000000002,"5min":0.27800000000000002,"15min":0.27100000000000002},"min":{"1min":0.23100000000000001,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.40400000000000003,"5min":0.47199999999999998,"15min":0.47199999999999998},"last":0.23999999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051796,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:43:55 2023","interfaces":[{"interface":"back","average":{"1min":0.26100000000000001,"5min":0.26700000000000002,"15min":0.27000000000000002},"min":{"1min":0.17799999999999999,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.374,"5min":0.44,"15min":0.44},"last":0.25600000000000001},{"interface":"front","average":{"1min":0.28299999999999997,"5min":0.28100000000000003,"15min":0.28399999999999997},"min":{"1min":0.20899999999999999,"5min":0.17599999999999999,"15min":0.16600000000000001},"max":{"1min":0.40100000000000002,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.28000000000000003}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:45:05.762 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:45:06.073 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all

Ignoring malparsed backtrace: 2023-03-25T04:45:25.658 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":624,"stamp":"2023-03-25T04:45:24.304294+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002366"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":41,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:42:34.382196+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:42:34.382196+0000","last_peered":"2023-03-25T04:42:34.382196+0000","last_clean":"2023-03-25T04:42:34.382196+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:42:34.382196+0000","last_undegraded":"2023-03-25T04:42:34.382196+0000","last_fullsized":"2023-03-25T04:42:34.382196+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:42:29.381744+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:42:29.381744+0000","last_peered":"2023-03-25T04:42:29.381744+0000","last_clean":"2023-03-25T04:42:29.381744+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:42:29.381744+0000","last_undegraded":"2023-03-25T04:42:29.381744+0000","last_fullsized":"2023-03-25T04:42:29.381744+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:23.375106+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:41:23.375106+0000","last_peered":"2023-03-25T04:41:23.375106+0000","last_clean":"2023-03-25T04:41:23.375106+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:41:23.375106+0000","last_undegraded":"2023-03-25T04:41:23.375106+0000","last_fullsized":"2023-03-25T04:41:23.375106+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:50.618930+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:41:50.618930+0000","last_peered":"2023-03-25T04:41:50.618930+0000","last_clean":"2023-03-25T04:41:50.618930+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:41:50.618930+0000","last_undegraded":"2023-03-25T04:41:50.618930+0000","last_fullsized":"2023-03-25T04:41:50.618930+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":41,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:58.377821+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:41:58.377821+0000","last_peered":"2023-03-25T04:41:58.377821+0000","last_clean":"2023-03-25T04:41:58.377821+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:41:58.377821+0000","last_undegraded":"2023-03-25T04:41:58.377821+0000","last_fullsized":"2023-03-25T04:41:58.377821+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:58.377836+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:41:58.377836+0000","last_peered":"2023-03-25T04:41:58.377836+0000","last_clean":"2023-03-25T04:41:58.377836+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:41:58.377836+0000","last_undegraded":"2023-03-25T04:41:58.377836+0000","last_fullsized":"2023-03-25T04:41:58.377836+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117213,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:45:01 2023","interfaces":[{"interface":"back","average":{"1min":0.251,"5min":0.26800000000000002,"15min":0.26400000000000001},"min":{"1min":0.183,"5min":0.17499999999999999,"15min":0.16},"max":{"1min":0.41499999999999998,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.26900000000000002},{"interface":"front","average":{"1min":0.28000000000000003,"5min":0.27900000000000003,"15min":0.27100000000000002},"min":{"1min":0.20999999999999999,"5min":0.185,"15min":0.17499999999999999},"max":{"1min":0.44400000000000001,"5min":0.47199999999999998,"15min":0.47199999999999998},"last":0.27900000000000003}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051803,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:44:55 2023","interfaces":[{"interface":"back","average":{"1min":0.252,"5min":0.26500000000000001,"15min":0.26900000000000002},"min":{"1min":0.183,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.39700000000000002,"5min":0.44,"15min":0.44},"last":0.20699999999999999},{"interface":"front","average":{"1min":0.27000000000000002,"5min":0.28100000000000003,"15min":0.28299999999999997},"min":{"1min":0.20300000000000001,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":0.42099999999999999,"5min":0.47599999999999998,"15min":0.48599999999999999},"last":0.23599999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:45:25.659 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:45:25.969 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:45:42.735+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:45:42.744+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:45:42.897 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace: 2023-03-25T04:46:00.183 INFO:teuthology.orchestra.run.smithi070.stdout:
, 2023-03-25T04:46:00.183 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":1217,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":20,"modified":"2023-03-25T04:45:42.307700+0000","services":{"mgr":{"daemons":{"summary":"","x":{"start_epoch":0,"start_stamp":"0.000000","gid":0,"addr":"(unrecognized address family 0)/0","metadata":{},"task_status":{}}}}}},"progress_events":{}}
, 2023-03-25T04:46:00.197 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now

Ignoring malparsed backtrace: 2023-03-25T04:46:19.910+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:46:20.039 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all

Ignoring malparsed backtrace: 2023-03-25T04:46:39.611 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}
, 2023-03-25T04:46:39.611 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json
, 2023-03-25T04:46:39.613+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:46:58.485+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:46:58.764+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:46:59.519+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:47:14.132 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:47:14.146 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":678,"stamp":"2023-03-25T04:47:12.325044+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002571"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":42,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:44.393426+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:44:44.393426+0000","last_peered":"2023-03-25T04:44:44.393426+0000","last_clean":"2023-03-25T04:44:44.393426+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:44:44.393426+0000","last_undegraded":"2023-03-25T04:44:44.393426+0000","last_fullsized":"2023-03-25T04:44:44.393426+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:39.393044+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:44:39.393044+0000","last_peered":"2023-03-25T04:44:39.393044+0000","last_clean":"2023-03-25T04:44:39.393044+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:44:39.393044+0000","last_undegraded":"2023-03-25T04:44:39.393044+0000","last_fullsized":"2023-03-25T04:44:39.393044+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:43:24.386177+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:43:24.386177+0000","last_peered":"2023-03-25T04:43:24.386177+0000","last_clean":"2023-03-25T04:43:24.386177+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:43:24.386177+0000","last_undegraded":"2023-03-25T04:43:24.386177+0000","last_fullsized":"2023-03-25T04:43:24.386177+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:43:55.629798+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:43:55.629798+0000","last_peered":"2023-03-25T04:43:55.629798+0000","last_clean":"2023-03-25T04:43:55.629798+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:43:55.629798+0000","last_undegraded":"2023-03-25T04:43:55.629798+0000","last_fullsized":"2023-03-25T04:43:55.629798+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":42,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:04.389904+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:44:04.389904+0000","last_peered":"2023-03-25T04:44:04.389904+0000","last_clean":"2023-03-25T04:44:04.389904+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:44:04.389904+0000","last_undegraded":"2023-03-25T04:44:04.389904+0000","last_fullsized":"2023-03-25T04:44:04.389904+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:04.389923+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:44:04.389923+0000","last_peered":"2023-03-25T04:44:04.389923+0000","last_clean":"2023-03-25T04:44:04.389923+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:44:04.389923+0000","last_undegraded":"2023-03-25T04:44:04.389923+0000","last_fullsized":"2023-03-25T04:44:04.389923+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117235,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:47:05 2023","interfaces":[{"interface":"back","average":{"1min":0.27800000000000002,"5min":0.27000000000000002,"15min":0.26300000000000001},"min":{"1min":0.20100000000000001,"5min":0.16900000000000001,"15min":0.16},"max":{"1min":0.39200000000000002,"5min":0.42999999999999999,"15min":0.45600000000000002},"last":0.379},{"interface":"front","average":{"1min":0.28399999999999997,"5min":0.28100000000000003,"15min":0.27200000000000002},"min":{"1min":0.17799999999999999,"5min":0.17799999999999999,"15min":0.17699999999999999},"max":{"1min":0.42599999999999999,"5min":0.47199999999999998,"15min":0.47199999999999998},"last":0.39800000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051825,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:47:01 2023","interfaces":[{"interface":"back","average":{"1min":0.27300000000000002,"5min":0.26100000000000001,"15min":0.27000000000000002},"min":{"1min":0.186,"5min":0.17399999999999999,"15min":0.17000000000000001},"max":{"1min":0.40400000000000003,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.23899999999999999},{"interface":"front","average":{"1min":0.28699999999999998,"5min":0.27500000000000002,"15min":0.28299999999999997},"min":{"1min":0.186,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.48599999999999999},"last":0.19600000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:47:31.198 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json
, 2023-03-25T04:47:31.411+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:47:31.789 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:47:49.509 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":696,"stamp":"2023-03-25T04:47:48.332093+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002697"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":42,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:44.393426+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:44:44.393426+0000","last_peered":"2023-03-25T04:44:44.393426+0000","last_clean":"2023-03-25T04:44:44.393426+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:44:44.393426+0000","last_undegraded":"2023-03-25T04:44:44.393426+0000","last_fullsized":"2023-03-25T04:44:44.393426+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:39.393044+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:44:39.393044+0000","last_peered":"2023-03-25T04:44:39.393044+0000","last_clean":"2023-03-25T04:44:39.393044+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:44:39.393044+0000","last_undegraded":"2023-03-25T04:44:39.393044+0000","last_fullsized":"2023-03-25T04:44:39.393044+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:45:24.396717+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:45:24.396717+0000","last_peered":"2023-03-25T04:45:24.396717+0000","last_clean":"2023-03-25T04:45:24.396717+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:45:24.396717+0000","last_undegraded":"2023-03-25T04:45:24.396717+0000","last_fullsized":"2023-03-25T04:45:24.396717+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:43:55.629798+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:43:55.629798+0000","last_peered":"2023-03-25T04:43:55.629798+0000","last_clean":"2023-03-25T04:43:55.629798+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:43:55.629798+0000","last_undegraded":"2023-03-25T04:43:55.629798+0000","last_fullsized":"2023-03-25T04:43:55.629798+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":42,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:04.389904+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:44:04.389904+0000","last_peered":"2023-03-25T04:44:04.389904+0000","last_clean":"2023-03-25T04:44:04.389904+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:44:04.389904+0000","last_undegraded":"2023-03-25T04:44:04.389904+0000","last_fullsized":"2023-03-25T04:44:04.389904+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:04.389923+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:44:04.389923+0000","last_peered":"2023-03-25T04:44:04.389923+0000","last_clean":"2023-03-25T04:44:04.389923+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:44:04.389923+0000","last_undegraded":"2023-03-25T04:44:04.389923+0000","last_fullsized":"2023-03-25T04:44:04.389923+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117242,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:47:05 2023","interfaces":[{"interface":"back","average":{"1min":0.27800000000000002,"5min":0.27000000000000002,"15min":0.26300000000000001},"min":{"1min":0.20100000000000001,"5min":0.16900000000000001,"15min":0.16},"max":{"1min":0.39200000000000002,"5min":0.42999999999999999,"15min":0.45600000000000002},"last":0.40500000000000003},{"interface":"front","average":{"1min":0.28399999999999997,"5min":0.28100000000000003,"15min":0.27200000000000002},"min":{"1min":0.17799999999999999,"5min":0.17799999999999999,"15min":0.17699999999999999},"max":{"1min":0.42599999999999999,"5min":0.47199999999999998,"15min":0.47199999999999998},"last":0.42399999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051834,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:47:01 2023","interfaces":[{"interface":"back","average":{"1min":0.27300000000000002,"5min":0.26100000000000001,"15min":0.27000000000000002},"min":{"1min":0.186,"5min":0.17399999999999999,"15min":0.17000000000000001},"max":{"1min":0.40400000000000003,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.25600000000000001},{"interface":"front","average":{"1min":0.28699999999999998,"5min":0.27500000000000002,"15min":0.28299999999999997},"min":{"1min":0.186,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.48599999999999999},"last":0.22800000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:47:49.510 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace:     ctx.managers[config['cluster']].wait_for_clean()
,   File "/home/teuthworker/src/github.com_ceph_ceph-c_511015c23eef0684a64171b710e432448df2b7c7/qa/tasks/ceph_manager.py", line 2747, in wait_for_clean
,     assert time.time() - start < timeout, \

Ignoring malparsed backtrace:  ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
,  1: /lib64/libpthread.so.0(+0x12cf0) [0x7f1453c2fcf0]
,  2: __pthread_mutex_lock()

Ignoring malparsed backtrace: 2023-03-25T04:28:05.172 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":142,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":5,"modified":"2023-03-25T04:27:12.092098+0000","services":{}},"progress_events":{}}
, 2023-03-25T04:28:05.185 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now
, 2023-03-25T04:28:05.186 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace: 2023-03-25T04:28:28.360 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":116,"stamp":"2023-03-25T04:28:28.108007+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002578"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.314661+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:27:45.314661+0000","last_peered":"2023-03-25T04:27:45.314661+0000","last_clean":"2023-03-25T04:27:45.314661+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:27:45.314661+0000","last_undegraded":"2023-03-25T04:27:45.314661+0000","last_fullsized":"2023-03-25T04:27:45.314661+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":25,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.310745+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:27:44.310745+0000","last_peered":"2023-03-25T04:27:44.310745+0000","last_clean":"2023-03-25T04:27:44.310745+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:27:44.310745+0000","last_undegraded":"2023-03-25T04:27:44.310745+0000","last_fullsized":"2023-03-25T04:27:44.310745+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":25,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.310650+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:27:44.310650+0000","last_peered":"2023-03-25T04:27:44.310650+0000","last_clean":"2023-03-25T04:27:44.310650+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:27:44.310650+0000","last_undegraded":"2023-03-25T04:27:44.310650+0000","last_fullsized":"2023-03-25T04:27:44.310650+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":26,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.312962+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:27:44.312962+0000","last_peered":"2023-03-25T04:27:44.312962+0000","last_clean":"2023-03-25T04:27:44.312962+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:27:44.312962+0000","last_undegraded":"2023-03-25T04:27:44.312962+0000","last_fullsized":"2023-03-25T04:27:44.312962+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315064+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:27:45.315064+0000","last_peered":"2023-03-25T04:27:45.315064+0000","last_clean":"2023-03-25T04:27:45.315064+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:27:45.315064+0000","last_undegraded":"2023-03-25T04:27:45.315064+0000","last_fullsized":"2023-03-25T04:27:45.315064+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":25,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.310707+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:27:44.310707+0000","last_peered":"2023-03-25T04:27:44.310707+0000","last_clean":"2023-03-25T04:27:44.310707+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:27:44.310707+0000","last_undegraded":"2023-03-25T04:27:44.310707+0000","last_fullsized":"2023-03-25T04:27:44.310707+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117007,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.375}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051597,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.186}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:28:28.362 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean
, 2023-03-25T04:28:28.362 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:28:45.655 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":124,"stamp":"2023-03-25T04:28:44.111265+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002492"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.314661+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:27:45.314661+0000","last_peered":"2023-03-25T04:27:45.314661+0000","last_clean":"2023-03-25T04:27:45.314661+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:27:45.314661+0000","last_undegraded":"2023-03-25T04:27:45.314661+0000","last_fullsized":"2023-03-25T04:27:45.314661+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":25,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.310745+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:27:44.310745+0000","last_peered":"2023-03-25T04:27:44.310745+0000","last_clean":"2023-03-25T04:27:44.310745+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:27:44.310745+0000","last_undegraded":"2023-03-25T04:27:44.310745+0000","last_fullsized":"2023-03-25T04:27:44.310745+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":25,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.310650+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:27:44.310650+0000","last_peered":"2023-03-25T04:27:44.310650+0000","last_clean":"2023-03-25T04:27:44.310650+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:27:44.310650+0000","last_undegraded":"2023-03-25T04:27:44.310650+0000","last_fullsized":"2023-03-25T04:27:44.310650+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":26,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.312962+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:27:44.312962+0000","last_peered":"2023-03-25T04:27:44.312962+0000","last_clean":"2023-03-25T04:27:44.312962+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:27:44.312962+0000","last_undegraded":"2023-03-25T04:27:44.312962+0000","last_fullsized":"2023-03-25T04:27:44.312962+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315064+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:27:45.315064+0000","last_peered":"2023-03-25T04:27:45.315064+0000","last_clean":"2023-03-25T04:27:45.315064+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:27:45.315064+0000","last_undegraded":"2023-03-25T04:27:45.315064+0000","last_fullsized":"2023-03-25T04:27:45.315064+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":25,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.310707+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:27:44.310707+0000","last_peered":"2023-03-25T04:27:44.310707+0000","last_clean":"2023-03-25T04:27:44.310707+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:27:44.310707+0000","last_undegraded":"2023-03-25T04:27:44.310707+0000","last_fullsized":"2023-03-25T04:27:44.310707+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117011,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:28:31 2023","interfaces":[{"interface":"back","average":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.36899999999999999},{"interface":"front","average":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"min":{"1min":0.19,"5min":0.19,"15min":0.19},"max":{"1min":0.46300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.39800000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051601,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:28:24 2023","interfaces":[{"interface":"back","average":{"1min":0.249,"5min":0.249,"15min":0.249},"min":{"1min":0.17699999999999999,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"last":0.254},{"interface":"front","average":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"min":{"1min":0.20200000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"last":0.312}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:28:45.656 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace: 2023-03-25T04:29:05.171 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}
, 2023-03-25T04:29:05.171 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json
, 2023-03-25T04:29:05.561 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:29:22.465 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:29:22.773 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all

Ignoring malparsed backtrace: 2023-03-25T04:29:42.405 INFO:teuthology.orchestra.run.smithi070.stdout:
, 2023-03-25T04:29:42.406 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":240,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":9,"modified":"2023-03-25T04:29:32.120358+0000","services":{"mon":{"daemons":{"summary":"","a":{"start_epoch":0,"start_stamp":"0.000000","gid":0,"addr":"(unrecognized address family 0)/0","metadata":{},"task_status":{}}}}}},"progress_events":{}}
, 2023-03-25T04:29:42.419 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:29:59.627 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":161,"stamp":"2023-03-25T04:29:58.125186+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002398"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315468+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:27:45.315468+0000","last_peered":"2023-03-25T04:27:45.315468+0000","last_clean":"2023-03-25T04:27:45.315468+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:27:45.315468+0000","last_undegraded":"2023-03-25T04:27:45.315468+0000","last_fullsized":"2023-03-25T04:27:45.315468+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312845+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:27:45.312845+0000","last_peered":"2023-03-25T04:27:45.312845+0000","last_clean":"2023-03-25T04:27:45.312845+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:27:45.312845+0000","last_undegraded":"2023-03-25T04:27:45.312845+0000","last_fullsized":"2023-03-25T04:27:45.312845+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312882+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:27:45.312882+0000","last_peered":"2023-03-25T04:27:45.312882+0000","last_clean":"2023-03-25T04:27:45.312882+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:27:45.312882+0000","last_undegraded":"2023-03-25T04:27:45.312882+0000","last_fullsized":"2023-03-25T04:27:45.312882+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.311730+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:27:45.311730+0000","last_peered":"2023-03-25T04:27:45.311730+0000","last_clean":"2023-03-25T04:27:45.311730+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:27:45.311730+0000","last_undegraded":"2023-03-25T04:27:45.311730+0000","last_fullsized":"2023-03-25T04:27:45.311730+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315735+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:27:45.315735+0000","last_peered":"2023-03-25T04:27:45.315735+0000","last_clean":"2023-03-25T04:27:45.315735+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:27:45.315735+0000","last_undegraded":"2023-03-25T04:27:45.315735+0000","last_fullsized":"2023-03-25T04:27:45.315735+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312833+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:27:45.312833+0000","last_peered":"2023-03-25T04:27:45.312833+0000","last_clean":"2023-03-25T04:27:45.312833+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:27:45.312833+0000","last_undegraded":"2023-03-25T04:27:45.312833+0000","last_fullsized":"2023-03-25T04:27:45.312833+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117026,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:29:33 2023","interfaces":[{"interface":"back","average":{"1min":0.26300000000000001,"5min":0.29099999999999998,"15min":0.29599999999999999},"min":{"1min":0.191,"5min":0.191,"15min":0.191},"max":{"1min":0.371,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.20499999999999999},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.29599999999999999,"15min":0.29999999999999999},"min":{"1min":0.21099999999999999,"5min":0.19,"15min":0.19},"max":{"1min":0.39800000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.19500000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051616,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:29:29 2023","interfaces":[{"interface":"back","average":{"1min":0.27200000000000002,"5min":0.253,"15min":0.25},"min":{"1min":0.183,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"last":0.26400000000000001},{"interface":"front","average":{"1min":0.28499999999999998,"5min":0.26800000000000002,"15min":0.26500000000000001},"min":{"1min":0.16600000000000001,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"last":0.29899999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:29:59.629 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean

Ignoring malparsed backtrace: 2023-03-25T04:30:16.889 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:30:16.903 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":170,"stamp":"2023-03-25T04:30:16.128427+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002222"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315468+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:27:45.315468+0000","last_peered":"2023-03-25T04:27:45.315468+0000","last_clean":"2023-03-25T04:27:45.315468+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:27:45.315468+0000","last_undegraded":"2023-03-25T04:27:45.315468+0000","last_fullsized":"2023-03-25T04:27:45.315468+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312845+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:27:45.312845+0000","last_peered":"2023-03-25T04:27:45.312845+0000","last_clean":"2023-03-25T04:27:45.312845+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:27:45.312845+0000","last_undegraded":"2023-03-25T04:27:45.312845+0000","last_fullsized":"2023-03-25T04:27:45.312845+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312882+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:27:45.312882+0000","last_peered":"2023-03-25T04:27:45.312882+0000","last_clean":"2023-03-25T04:27:45.312882+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:27:45.312882+0000","last_undegraded":"2023-03-25T04:27:45.312882+0000","last_fullsized":"2023-03-25T04:27:45.312882+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.311730+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:27:45.311730+0000","last_peered":"2023-03-25T04:27:45.311730+0000","last_clean":"2023-03-25T04:27:45.311730+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:27:45.311730+0000","last_undegraded":"2023-03-25T04:27:45.311730+0000","last_fullsized":"2023-03-25T04:27:45.311730+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315735+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:27:45.315735+0000","last_peered":"2023-03-25T04:27:45.315735+0000","last_clean":"2023-03-25T04:27:45.315735+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:27:45.315735+0000","last_undegraded":"2023-03-25T04:27:45.315735+0000","last_fullsized":"2023-03-25T04:27:45.315735+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312833+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:27:45.312833+0000","last_peered":"2023-03-25T04:27:45.312833+0000","last_clean":"2023-03-25T04:27:45.312833+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:27:45.312833+0000","last_undegraded":"2023-03-25T04:27:45.312833+0000","last_fullsized":"2023-03-25T04:27:45.312833+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117030,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:29:33 2023","interfaces":[{"interface":"back","average":{"1min":0.26300000000000001,"5min":0.29099999999999998,"15min":0.29599999999999999},"min":{"1min":0.191,"5min":0.191,"15min":0.191},"max":{"1min":0.371,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.245},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.29599999999999999,"15min":0.29999999999999999},"min":{"1min":0.21099999999999999,"5min":0.19,"15min":0.19},"max":{"1min":0.39800000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.20899999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051620,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:29:29 2023","interfaces":[{"interface":"back","average":{"1min":0.27200000000000002,"5min":0.253,"15min":0.25},"min":{"1min":0.183,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"last":0.19400000000000001},{"interface":"front","average":{"1min":0.28499999999999998,"5min":0.26800000000000002,"15min":0.26500000000000001},"min":{"1min":0.16600000000000001,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"last":0.23599999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:30:31.984+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:30:32.201+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:30:32.981+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)

Ignoring malparsed backtrace: 2023-03-25T04:30:48.449 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now
, 2023-03-25T04:30:48.450 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:30:48.760 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:31:08.342 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":196,"stamp":"2023-03-25T04:31:08.139018+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002431"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315468+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:27:45.315468+0000","last_peered":"2023-03-25T04:27:45.315468+0000","last_clean":"2023-03-25T04:27:45.315468+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:27:45.315468+0000","last_undegraded":"2023-03-25T04:27:45.315468+0000","last_fullsized":"2023-03-25T04:27:45.315468+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312845+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:27:45.312845+0000","last_peered":"2023-03-25T04:27:45.312845+0000","last_clean":"2023-03-25T04:27:45.312845+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:27:45.312845+0000","last_undegraded":"2023-03-25T04:27:45.312845+0000","last_fullsized":"2023-03-25T04:27:45.312845+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312882+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:27:45.312882+0000","last_peered":"2023-03-25T04:27:45.312882+0000","last_clean":"2023-03-25T04:27:45.312882+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:27:45.312882+0000","last_undegraded":"2023-03-25T04:27:45.312882+0000","last_fullsized":"2023-03-25T04:27:45.312882+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.311730+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:27:45.311730+0000","last_peered":"2023-03-25T04:27:45.311730+0000","last_clean":"2023-03-25T04:27:45.311730+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:27:45.311730+0000","last_undegraded":"2023-03-25T04:27:45.311730+0000","last_fullsized":"2023-03-25T04:27:45.311730+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315735+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:27:45.315735+0000","last_peered":"2023-03-25T04:27:45.315735+0000","last_clean":"2023-03-25T04:27:45.315735+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:27:45.315735+0000","last_undegraded":"2023-03-25T04:27:45.315735+0000","last_fullsized":"2023-03-25T04:27:45.315735+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312833+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:27:45.312833+0000","last_peered":"2023-03-25T04:27:45.312833+0000","last_clean":"2023-03-25T04:27:45.312833+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:27:45.312833+0000","last_undegraded":"2023-03-25T04:27:45.312833+0000","last_fullsized":"2023-03-25T04:27:45.312833+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117040,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:30:37 2023","interfaces":[{"interface":"back","average":{"1min":0.28799999999999998,"5min":0.28899999999999998,"15min":0.29499999999999998},"min":{"1min":0.20200000000000001,"5min":0.191,"15min":0.191},"max":{"1min":0.433,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.29799999999999999},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.28999999999999998,"15min":0.29799999999999999},"min":{"1min":0.17499999999999999,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":0.40300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.25700000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051630,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:30:32 2023","interfaces":[{"interface":"back","average":{"1min":0.25600000000000001,"5min":0.255,"15min":0.251},"min":{"1min":0.19400000000000001,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.35699999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"last":0.22800000000000001},{"interface":"front","average":{"1min":0.28699999999999998,"5min":0.27200000000000002,"15min":0.26600000000000001},"min":{"1min":0.20599999999999999,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.39400000000000002,"5min":0.42899999999999999,"15min":0.42899999999999999},"last":0.24099999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:31:08.343 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace: 2023-03-25T04:31:24.283+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:31:25.192+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:31:25.295+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)

Ignoring malparsed backtrace: 2023-03-25T04:31:40.257 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:31:40.271 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":212,"stamp":"2023-03-25T04:31:40.144732+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002208"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315468+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:27:45.315468+0000","last_peered":"2023-03-25T04:27:45.315468+0000","last_clean":"2023-03-25T04:27:45.315468+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:27:45.315468+0000","last_undegraded":"2023-03-25T04:27:45.315468+0000","last_fullsized":"2023-03-25T04:27:45.315468+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312845+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:27:45.312845+0000","last_peered":"2023-03-25T04:27:45.312845+0000","last_clean":"2023-03-25T04:27:45.312845+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:27:45.312845+0000","last_undegraded":"2023-03-25T04:27:45.312845+0000","last_fullsized":"2023-03-25T04:27:45.312845+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:23.308607+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:29:23.308607+0000","last_peered":"2023-03-25T04:29:23.308607+0000","last_clean":"2023-03-25T04:29:23.308607+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:29:23.308607+0000","last_undegraded":"2023-03-25T04:29:23.308607+0000","last_fullsized":"2023-03-25T04:29:23.308607+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":28,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:20.554710+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:29:20.554710+0000","last_peered":"2023-03-25T04:29:20.554710+0000","last_clean":"2023-03-25T04:29:20.554710+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:29:20.554710+0000","last_undegraded":"2023-03-25T04:29:20.554710+0000","last_fullsized":"2023-03-25T04:29:20.554710+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309216+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:29:28.309216+0000","last_peered":"2023-03-25T04:29:28.309216+0000","last_clean":"2023-03-25T04:29:28.309216+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:29:28.309216+0000","last_undegraded":"2023-03-25T04:29:28.309216+0000","last_fullsized":"2023-03-25T04:29:28.309216+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309233+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:29:28.309233+0000","last_peered":"2023-03-25T04:29:28.309233+0000","last_clean":"2023-03-25T04:29:28.309233+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:29:28.309233+0000","last_undegraded":"2023-03-25T04:29:28.309233+0000","last_fullsized":"2023-03-25T04:29:28.309233+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117046,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:30:37 2023","interfaces":[{"interface":"back","average":{"1min":0.28799999999999998,"5min":0.28899999999999998,"15min":0.29499999999999998},"min":{"1min":0.20200000000000001,"5min":0.191,"15min":0.191},"max":{"1min":0.433,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.216},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.28999999999999998,"15min":0.29799999999999999},"min":{"1min":0.17499999999999999,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":0.40300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.24299999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051637,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:31:35 2023","interfaces":[{"interface":"back","average":{"1min":0.26100000000000001,"5min":0.25700000000000001,"15min":0.251},"min":{"1min":0.19,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.35899999999999999,"5min":0.41599999999999998,"15min":0.41599999999999998},"last":0.221},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.27400000000000002,"15min":0.26700000000000002},"min":{"1min":0.20200000000000001,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.374,"5min":0.42899999999999999,"15min":0.42899999999999999},"last":0.24099999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:32:01.906+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:32:02.388+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:32:02.794 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:32:22.326 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":233,"stamp":"2023-03-25T04:32:22.153289+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002545"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:33.309748+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:29:33.309748+0000","last_peered":"2023-03-25T04:29:33.309748+0000","last_clean":"2023-03-25T04:29:33.309748+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:29:33.309748+0000","last_undegraded":"2023-03-25T04:29:33.309748+0000","last_fullsized":"2023-03-25T04:29:33.309748+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:33.309729+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:29:33.309729+0000","last_peered":"2023-03-25T04:29:33.309729+0000","last_clean":"2023-03-25T04:29:33.309729+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:29:33.309729+0000","last_undegraded":"2023-03-25T04:29:33.309729+0000","last_fullsized":"2023-03-25T04:29:33.309729+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:23.308607+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:29:23.308607+0000","last_peered":"2023-03-25T04:29:23.308607+0000","last_clean":"2023-03-25T04:29:23.308607+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:29:23.308607+0000","last_undegraded":"2023-03-25T04:29:23.308607+0000","last_fullsized":"2023-03-25T04:29:23.308607+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":28,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:20.554710+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:29:20.554710+0000","last_peered":"2023-03-25T04:29:20.554710+0000","last_clean":"2023-03-25T04:29:20.554710+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:29:20.554710+0000","last_undegraded":"2023-03-25T04:29:20.554710+0000","last_fullsized":"2023-03-25T04:29:20.554710+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309216+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:29:28.309216+0000","last_peered":"2023-03-25T04:29:28.309216+0000","last_clean":"2023-03-25T04:29:28.309216+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:29:28.309216+0000","last_undegraded":"2023-03-25T04:29:28.309216+0000","last_fullsized":"2023-03-25T04:29:28.309216+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309233+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:29:28.309233+0000","last_peered":"2023-03-25T04:29:28.309233+0000","last_clean":"2023-03-25T04:29:28.309233+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:29:28.309233+0000","last_undegraded":"2023-03-25T04:29:28.309233+0000","last_fullsized":"2023-03-25T04:29:28.309233+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117055,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:31:42 2023","interfaces":[{"interface":"back","average":{"1min":0.27500000000000002,"5min":0.28399999999999997,"15min":0.29399999999999998},"min":{"1min":0.19800000000000001,"5min":0.191,"15min":0.191},"max":{"1min":0.36899999999999999,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.32300000000000001},{"interface":"front","average":{"1min":0.27900000000000003,"5min":0.28499999999999998,"15min":0.29599999999999999},"min":{"1min":0.216,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":0.40799999999999997,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.27100000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051646,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:31:35 2023","interfaces":[{"interface":"back","average":{"1min":0.26100000000000001,"5min":0.25700000000000001,"15min":0.251},"min":{"1min":0.19,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.35899999999999999,"5min":0.41599999999999998,"15min":0.41599999999999998},"last":0.25600000000000001},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.27400000000000002,"15min":0.26700000000000002},"min":{"1min":0.20200000000000001,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.374,"5min":0.42899999999999999,"15min":0.42899999999999999},"last":0.29199999999999998}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:32:22.327 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean
, 2023-03-25T04:32:22.327 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}

Ignoring malparsed backtrace: 2023-03-25T04:32:39.836 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:32:39.849 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":241,"stamp":"2023-03-25T04:32:38.156312+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002346"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:33.309748+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:29:33.309748+0000","last_peered":"2023-03-25T04:29:33.309748+0000","last_clean":"2023-03-25T04:29:33.309748+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:29:33.309748+0000","last_undegraded":"2023-03-25T04:29:33.309748+0000","last_fullsized":"2023-03-25T04:29:33.309748+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:33.309729+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:29:33.309729+0000","last_peered":"2023-03-25T04:29:33.309729+0000","last_clean":"2023-03-25T04:29:33.309729+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:29:33.309729+0000","last_undegraded":"2023-03-25T04:29:33.309729+0000","last_fullsized":"2023-03-25T04:29:33.309729+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:23.308607+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:29:23.308607+0000","last_peered":"2023-03-25T04:29:23.308607+0000","last_clean":"2023-03-25T04:29:23.308607+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:29:23.308607+0000","last_undegraded":"2023-03-25T04:29:23.308607+0000","last_fullsized":"2023-03-25T04:29:23.308607+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":28,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:20.554710+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:29:20.554710+0000","last_peered":"2023-03-25T04:29:20.554710+0000","last_clean":"2023-03-25T04:29:20.554710+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:29:20.554710+0000","last_undegraded":"2023-03-25T04:29:20.554710+0000","last_fullsized":"2023-03-25T04:29:20.554710+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309216+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:29:28.309216+0000","last_peered":"2023-03-25T04:29:28.309216+0000","last_clean":"2023-03-25T04:29:28.309216+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:29:28.309216+0000","last_undegraded":"2023-03-25T04:29:28.309216+0000","last_fullsized":"2023-03-25T04:29:28.309216+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309233+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:29:28.309233+0000","last_peered":"2023-03-25T04:29:28.309233+0000","last_clean":"2023-03-25T04:29:28.309233+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:29:28.309233+0000","last_undegraded":"2023-03-25T04:29:28.309233+0000","last_fullsized":"2023-03-25T04:29:28.309233+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117058,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:31:42 2023","interfaces":[{"interface":"back","average":{"1min":0.27500000000000002,"5min":0.28399999999999997,"15min":0.29399999999999998},"min":{"1min":0.19800000000000001,"5min":0.191,"15min":0.191},"max":{"1min":0.36899999999999999,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.30599999999999999},{"interface":"front","average":{"1min":0.27900000000000003,"5min":0.28499999999999998,"15min":0.29599999999999999},"min":{"1min":0.216,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":0.40799999999999997,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.32500000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051649,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:31:35 2023","interfaces":[{"interface":"back","average":{"1min":0.26100000000000001,"5min":0.25700000000000001,"15min":0.251},"min":{"1min":0.19,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.35899999999999999,"5min":0.41599999999999998,"15min":0.41599999999999998},"last":0.24199999999999999},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.27400000000000002,"15min":0.26700000000000002},"min":{"1min":0.20200000000000001,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.374,"5min":0.42899999999999999,"15min":0.42899999999999999},"last":0.22600000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:32:54.944+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:32:55.469+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:32:55.927+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)

Ignoring malparsed backtrace: 2023-03-25T04:33:13.958 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean
, 2023-03-25T04:33:13.959 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}
, 2023-03-25T04:33:13.959 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json

Ignoring malparsed backtrace: 2023-03-25T04:33:31.199 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":267,"stamp":"2023-03-25T04:33:30.166660+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002269"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:33.309748+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:29:33.309748+0000","last_peered":"2023-03-25T04:29:33.309748+0000","last_clean":"2023-03-25T04:29:33.309748+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:29:33.309748+0000","last_undegraded":"2023-03-25T04:29:33.309748+0000","last_fullsized":"2023-03-25T04:29:33.309748+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:33.309729+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:29:33.309729+0000","last_peered":"2023-03-25T04:29:33.309729+0000","last_clean":"2023-03-25T04:29:33.309729+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:29:33.309729+0000","last_undegraded":"2023-03-25T04:29:33.309729+0000","last_fullsized":"2023-03-25T04:29:33.309729+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":28,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:31:23.320894+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:31:23.320894+0000","last_peered":"2023-03-25T04:31:23.320894+0000","last_clean":"2023-03-25T04:31:23.320894+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:31:23.320894+0000","last_undegraded":"2023-03-25T04:31:23.320894+0000","last_fullsized":"2023-03-25T04:31:23.320894+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":28,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:20.554710+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:29:20.554710+0000","last_peered":"2023-03-25T04:29:20.554710+0000","last_clean":"2023-03-25T04:29:20.554710+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:29:20.554710+0000","last_undegraded":"2023-03-25T04:29:20.554710+0000","last_fullsized":"2023-03-25T04:29:20.554710+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309216+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:29:28.309216+0000","last_peered":"2023-03-25T04:29:28.309216+0000","last_clean":"2023-03-25T04:29:28.309216+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:29:28.309216+0000","last_undegraded":"2023-03-25T04:29:28.309216+0000","last_fullsized":"2023-03-25T04:29:28.309216+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309233+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:29:28.309233+0000","last_peered":"2023-03-25T04:29:28.309233+0000","last_clean":"2023-03-25T04:29:28.309233+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:29:28.309233+0000","last_undegraded":"2023-03-25T04:29:28.309233+0000","last_fullsized":"2023-03-25T04:29:28.309233+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117068,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:32:46 2023","interfaces":[{"interface":"back","average":{"1min":0.27000000000000002,"5min":0.27900000000000003,"15min":0.29199999999999998},"min":{"1min":0.17899999999999999,"5min":0.17899999999999999,"15min":0.17899999999999999},"max":{"1min":0.35599999999999998,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.28799999999999998},{"interface":"front","average":{"1min":0.26800000000000002,"5min":0.27800000000000002,"15min":0.29399999999999998},"min":{"1min":0.19900000000000001,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":0.377,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.315}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051660,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:32:36 2023","interfaces":[{"interface":"back","average":{"1min":0.248,"5min":0.25700000000000001,"15min":0.251},"min":{"1min":0.17000000000000001,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"last":0.22700000000000001},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.27600000000000002,"15min":0.26800000000000002},"min":{"1min":0.192,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"last":0.20399999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:33:31.201 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:33:31.400+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)

Ignoring malparsed backtrace: 2023-03-25T04:33:47.531+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:33:47.951+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:33:48.505+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)

Ignoring malparsed backtrace: 2023-03-25T04:34:06.104 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json
, 2023-03-25T04:34:06.118+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:34:06.497 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:34:23.399 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:34:23.665+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:34:23.712 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:34:40.195+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:34:40.523+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:34:40.676 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace: 2023-03-25T04:35:03.381+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:35:04.139+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:35:04.391+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)

Ignoring malparsed backtrace: 2023-03-25T04:35:22.738 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean
, 2023-03-25T04:35:22.739 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}
, 2023-03-25T04:35:22.739 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json

Ignoring malparsed backtrace: 2023-03-25T04:35:41.393+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:35:42.352+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:35:42.376+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:35:57.384 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":340,"stamp":"2023-03-25T04:35:56.194992+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.001930"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":36,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:31:43.322894+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:31:43.322894+0000","last_peered":"2023-03-25T04:31:43.322894+0000","last_clean":"2023-03-25T04:31:43.322894+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:31:43.322894+0000","last_undegraded":"2023-03-25T04:31:43.322894+0000","last_fullsized":"2023-03-25T04:31:43.322894+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":28,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:31:43.322876+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:31:43.322876+0000","last_peered":"2023-03-25T04:31:43.322876+0000","last_clean":"2023-03-25T04:31:43.322876+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:31:43.322876+0000","last_undegraded":"2023-03-25T04:31:43.322876+0000","last_fullsized":"2023-03-25T04:31:43.322876+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:23.331998+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:33:23.331998+0000","last_peered":"2023-03-25T04:33:23.331998+0000","last_clean":"2023-03-25T04:33:23.331998+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:33:23.331998+0000","last_undegraded":"2023-03-25T04:33:23.331998+0000","last_fullsized":"2023-03-25T04:33:23.331998+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:30.576723+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:33:30.576723+0000","last_peered":"2023-03-25T04:33:30.576723+0000","last_clean":"2023-03-25T04:33:30.576723+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:33:30.576723+0000","last_undegraded":"2023-03-25T04:33:30.576723+0000","last_fullsized":"2023-03-25T04:33:30.576723+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":37,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:38.333447+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:33:38.333447+0000","last_peered":"2023-03-25T04:33:38.333447+0000","last_clean":"2023-03-25T04:33:38.333447+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:33:38.333447+0000","last_undegraded":"2023-03-25T04:33:38.333447+0000","last_fullsized":"2023-03-25T04:33:38.333447+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:38.333467+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:33:38.333467+0000","last_peered":"2023-03-25T04:33:38.333467+0000","last_clean":"2023-03-25T04:33:38.333467+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:33:38.333467+0000","last_undegraded":"2023-03-25T04:33:38.333467+0000","last_fullsized":"2023-03-25T04:33:38.333467+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117098,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:35:49 2023","interfaces":[{"interface":"back","average":{"1min":0.255,"5min":0.26000000000000001,"15min":0.28199999999999997},"min":{"1min":0.17899999999999999,"5min":0.16,"15min":0.16},"max":{"1min":0.36399999999999999,"5min":0.42599999999999999,"15min":0.45100000000000001},"last":0.26100000000000001},{"interface":"front","average":{"1min":0.26000000000000001,"5min":0.26700000000000002,"15min":0.28599999999999998},"min":{"1min":0.188,"5min":0.17999999999999999,"15min":0.17499999999999999},"max":{"1min":0.39100000000000001,"5min":0.45200000000000001,"15min":0.46300000000000002},"last":0.28399999999999997}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051689,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:35:42 2023","interfaces":[{"interface":"back","average":{"1min":0.29399999999999998,"5min":0.27200000000000002,"15min":0.25800000000000001},"min":{"1min":0.17299999999999999,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.41099999999999998,"5min":0.42699999999999999,"15min":0.42699999999999999},"last":0.20399999999999999},{"interface":"front","average":{"1min":0.30299999999999999,"5min":0.28399999999999997,"15min":0.27300000000000002},"min":{"1min":0.19700000000000001,"5min":0.192,"15min":0.16600000000000001},"max":{"1min":0.43099999999999999,"5min":0.45800000000000002,"15min":0.45800000000000002},"last":0.22}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:35:57.624+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)

Ignoring malparsed backtrace: 2023-03-25T04:36:17.226 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:36:17.239 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":350,"stamp":"2023-03-25T04:36:16.198866+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002148"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":37,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:53.335026+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:33:53.335026+0000","last_peered":"2023-03-25T04:33:53.335026+0000","last_clean":"2023-03-25T04:33:53.335026+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:33:53.335026+0000","last_undegraded":"2023-03-25T04:33:53.335026+0000","last_fullsized":"2023-03-25T04:33:53.335026+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:53.335009+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:33:53.335009+0000","last_peered":"2023-03-25T04:33:53.335009+0000","last_clean":"2023-03-25T04:33:53.335009+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:33:53.335009+0000","last_undegraded":"2023-03-25T04:33:53.335009+0000","last_fullsized":"2023-03-25T04:33:53.335009+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:23.331998+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:33:23.331998+0000","last_peered":"2023-03-25T04:33:23.331998+0000","last_clean":"2023-03-25T04:33:23.331998+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:33:23.331998+0000","last_undegraded":"2023-03-25T04:33:23.331998+0000","last_fullsized":"2023-03-25T04:33:23.331998+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:30.576723+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:33:30.576723+0000","last_peered":"2023-03-25T04:33:30.576723+0000","last_clean":"2023-03-25T04:33:30.576723+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:33:30.576723+0000","last_undegraded":"2023-03-25T04:33:30.576723+0000","last_fullsized":"2023-03-25T04:33:30.576723+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":37,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:38.333447+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:33:38.333447+0000","last_peered":"2023-03-25T04:33:38.333447+0000","last_clean":"2023-03-25T04:33:38.333447+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:33:38.333447+0000","last_undegraded":"2023-03-25T04:33:38.333447+0000","last_fullsized":"2023-03-25T04:33:38.333447+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:38.333467+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:33:38.333467+0000","last_peered":"2023-03-25T04:33:38.333467+0000","last_clean":"2023-03-25T04:33:38.333467+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:33:38.333467+0000","last_undegraded":"2023-03-25T04:33:38.333467+0000","last_fullsized":"2023-03-25T04:33:38.333467+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117102,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:35:49 2023","interfaces":[{"interface":"back","average":{"1min":0.255,"5min":0.26000000000000001,"15min":0.28199999999999997},"min":{"1min":0.17899999999999999,"5min":0.16,"15min":0.16},"max":{"1min":0.36399999999999999,"5min":0.42599999999999999,"15min":0.45100000000000001},"last":0.28399999999999997},{"interface":"front","average":{"1min":0.26000000000000001,"5min":0.26700000000000002,"15min":0.28599999999999998},"min":{"1min":0.188,"5min":0.17999999999999999,"15min":0.17499999999999999},"max":{"1min":0.39100000000000001,"5min":0.45200000000000001,"15min":0.46300000000000002},"last":0.30299999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051693,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:35:42 2023","interfaces":[{"interface":"back","average":{"1min":0.29399999999999998,"5min":0.27200000000000002,"15min":0.25800000000000001},"min":{"1min":0.17299999999999999,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.41099999999999998,"5min":0.42699999999999999,"15min":0.42699999999999999},"last":0.35099999999999998},{"interface":"front","average":{"1min":0.30299999999999999,"5min":0.28399999999999997,"15min":0.27300000000000002},"min":{"1min":0.19700000000000001,"5min":0.192,"15min":0.16600000000000001},"max":{"1min":0.43099999999999999,"5min":0.45800000000000002,"15min":0.45800000000000002},"last":0.32900000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:36:34.556+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:36:34.717+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:36:35.583+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)

Ignoring malparsed backtrace: 2023-03-25T04:36:52.680 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json
, 2023-03-25T04:36:52.701+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:36:53.067 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:37:09.963 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:37:10.273 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all

Ignoring malparsed backtrace: 2023-03-25T04:37:26.856+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:37:27.288 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:37:27.538+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:37:41.903 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":392,"stamp":"2023-03-25T04:37:40.213373+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002012"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":37,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:53.335026+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:33:53.335026+0000","last_peered":"2023-03-25T04:33:53.335026+0000","last_clean":"2023-03-25T04:33:53.335026+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:33:53.335026+0000","last_undegraded":"2023-03-25T04:33:53.335026+0000","last_fullsized":"2023-03-25T04:33:53.335026+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:53.335009+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:33:53.335009+0000","last_peered":"2023-03-25T04:33:53.335009+0000","last_clean":"2023-03-25T04:33:53.335009+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:33:53.335009+0000","last_undegraded":"2023-03-25T04:33:53.335009+0000","last_fullsized":"2023-03-25T04:33:53.335009+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:23.343585+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:35:23.343585+0000","last_peered":"2023-03-25T04:35:23.343585+0000","last_clean":"2023-03-25T04:35:23.343585+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:35:23.343585+0000","last_undegraded":"2023-03-25T04:35:23.343585+0000","last_fullsized":"2023-03-25T04:35:23.343585+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:30.576723+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:33:30.576723+0000","last_peered":"2023-03-25T04:33:30.576723+0000","last_clean":"2023-03-25T04:33:30.576723+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:33:30.576723+0000","last_undegraded":"2023-03-25T04:33:30.576723+0000","last_fullsized":"2023-03-25T04:33:30.576723+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":37,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:38.333447+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:33:38.333447+0000","last_peered":"2023-03-25T04:33:38.333447+0000","last_clean":"2023-03-25T04:33:38.333447+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:33:38.333447+0000","last_undegraded":"2023-03-25T04:33:38.333447+0000","last_fullsized":"2023-03-25T04:33:38.333447+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:38.333467+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:33:38.333467+0000","last_peered":"2023-03-25T04:33:38.333467+0000","last_clean":"2023-03-25T04:33:38.333467+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:33:38.333467+0000","last_undegraded":"2023-03-25T04:33:38.333467+0000","last_fullsized":"2023-03-25T04:33:38.333467+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117119,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:36:50 2023","interfaces":[{"interface":"back","average":{"1min":0.254,"5min":0.25600000000000001,"15min":0.27900000000000003},"min":{"1min":0.20499999999999999,"5min":0.16,"15min":0.16},"max":{"1min":0.371,"5min":0.42599999999999999,"15min":0.45100000000000001},"last":0.23000000000000001},{"interface":"front","average":{"1min":0.26800000000000002,"5min":0.26500000000000001,"15min":0.28399999999999997},"min":{"1min":0.19700000000000001,"5min":0.17999999999999999,"15min":0.17499999999999999},"max":{"1min":0.34699999999999998,"5min":0.45200000000000001,"15min":0.46300000000000002},"last":0.20899999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051710,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:36:44 2023","interfaces":[{"interface":"back","average":{"1min":0.28399999999999997,"5min":0.27700000000000002,"15min":0.26100000000000001},"min":{"1min":0.192,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.40899999999999997,"5min":0.42699999999999999,"15min":0.42699999999999999},"last":0.39700000000000002},{"interface":"front","average":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.27500000000000002},"min":{"1min":0.182,"5min":0.182,"15min":0.16600000000000001},"max":{"1min":0.439,"5min":0.45800000000000002,"15min":0.45800000000000002},"last":0.439}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:37:42.400+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)

Ignoring malparsed backtrace: 2023-03-25T04:38:04.766+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:38:04.922 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:38:05.246 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:38:19.605 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":411,"stamp":"2023-03-25T04:38:18.220966+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002368"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":38,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:36:03.347070+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:36:03.347070+0000","last_peered":"2023-03-25T04:36:03.347070+0000","last_clean":"2023-03-25T04:36:03.347070+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:36:03.347070+0000","last_undegraded":"2023-03-25T04:36:03.347070+0000","last_fullsized":"2023-03-25T04:36:03.347070+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:36:03.347025+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:36:03.347025+0000","last_peered":"2023-03-25T04:36:03.347025+0000","last_clean":"2023-03-25T04:36:03.347025+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:36:03.347025+0000","last_undegraded":"2023-03-25T04:36:03.347025+0000","last_fullsized":"2023-03-25T04:36:03.347025+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:23.343585+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:35:23.343585+0000","last_peered":"2023-03-25T04:35:23.343585+0000","last_clean":"2023-03-25T04:35:23.343585+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:35:23.343585+0000","last_undegraded":"2023-03-25T04:35:23.343585+0000","last_fullsized":"2023-03-25T04:35:23.343585+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:35.587025+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:35:35.587025+0000","last_peered":"2023-03-25T04:35:35.587025+0000","last_clean":"2023-03-25T04:35:35.587025+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:35:35.587025+0000","last_undegraded":"2023-03-25T04:35:35.587025+0000","last_fullsized":"2023-03-25T04:35:35.587025+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":38,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:43.345308+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:35:43.345308+0000","last_peered":"2023-03-25T04:35:43.345308+0000","last_clean":"2023-03-25T04:35:43.345308+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:35:43.345308+0000","last_undegraded":"2023-03-25T04:35:43.345308+0000","last_fullsized":"2023-03-25T04:35:43.345308+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:43.345324+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:35:43.345324+0000","last_peered":"2023-03-25T04:35:43.345324+0000","last_clean":"2023-03-25T04:35:43.345324+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:35:43.345324+0000","last_undegraded":"2023-03-25T04:35:43.345324+0000","last_fullsized":"2023-03-25T04:35:43.345324+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117127,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:37:53 2023","interfaces":[{"interface":"back","average":{"1min":0.25800000000000001,"5min":0.253,"15min":0.27700000000000002},"min":{"1min":0.184,"5min":0.16,"15min":0.16},"max":{"1min":0.44600000000000001,"5min":0.44600000000000001,"15min":0.45100000000000001},"last":0.35199999999999998},{"interface":"front","average":{"1min":0.26900000000000002,"5min":0.26500000000000001,"15min":0.28199999999999997},"min":{"1min":0.192,"5min":0.17999999999999999,"15min":0.17499999999999999},"max":{"1min":0.46400000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"last":0.33200000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051717,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:37:44 2023","interfaces":[{"interface":"back","average":{"1min":0.27300000000000002,"5min":0.28199999999999997,"15min":0.26200000000000001},"min":{"1min":0.17899999999999999,"5min":0.17299999999999999,"15min":0.17000000000000001},"max":{"1min":0.41799999999999998,"5min":0.42099999999999999,"15min":0.42699999999999999},"last":0.28699999999999998},{"interface":"front","average":{"1min":0.29399999999999998,"5min":0.29099999999999998,"15min":0.27700000000000002},"min":{"1min":0.19900000000000001,"5min":0.182,"15min":0.16600000000000001},"max":{"1min":0.441,"5min":0.441,"15min":0.45800000000000002},"last":0.26300000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:38:19.718+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:38:20.315+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)

Ignoring malparsed backtrace: 2023-03-25T04:38:44.283 INFO:teuthology.orchestra.run.smithi070.stdout:
, 2023-03-25T04:38:44.283 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":781,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":16,"modified":"2023-03-25T04:38:40.225321+0000","services":{}},"progress_events":{}}
, 2023-03-25T04:38:44.296 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now

Ignoring malparsed backtrace: 2023-03-25T04:39:02.477+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:39:02.578+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:39:03.524+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:39:20.359 INFO:teuthology.orchestra.run.smithi070.stdout:
, 2023-03-25T04:39:20.359 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":818,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":16,"modified":"2023-03-25T04:38:40.225321+0000","services":{}},"progress_events":{}}
, 2023-03-25T04:39:20.372 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now

Ignoring malparsed backtrace: 2023-03-25T04:39:37.527 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:39:37.541 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":450,"stamp":"2023-03-25T04:39:36.235332+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002086"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":38,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:36:03.347070+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:36:03.347070+0000","last_peered":"2023-03-25T04:36:03.347070+0000","last_clean":"2023-03-25T04:36:03.347070+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:36:03.347070+0000","last_undegraded":"2023-03-25T04:36:03.347070+0000","last_fullsized":"2023-03-25T04:36:03.347070+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:36:03.347025+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:36:03.347025+0000","last_peered":"2023-03-25T04:36:03.347025+0000","last_clean":"2023-03-25T04:36:03.347025+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:36:03.347025+0000","last_undegraded":"2023-03-25T04:36:03.347025+0000","last_fullsized":"2023-03-25T04:36:03.347025+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:23.353778+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:37:23.353778+0000","last_peered":"2023-03-25T04:37:23.353778+0000","last_clean":"2023-03-25T04:37:23.353778+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:37:23.353778+0000","last_undegraded":"2023-03-25T04:37:23.353778+0000","last_fullsized":"2023-03-25T04:37:23.353778+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:35.587025+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:35:35.587025+0000","last_peered":"2023-03-25T04:35:35.587025+0000","last_clean":"2023-03-25T04:35:35.587025+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:35:35.587025+0000","last_undegraded":"2023-03-25T04:35:35.587025+0000","last_fullsized":"2023-03-25T04:35:35.587025+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":38,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:43.345308+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:35:43.345308+0000","last_peered":"2023-03-25T04:35:43.345308+0000","last_clean":"2023-03-25T04:35:43.345308+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:35:43.345308+0000","last_undegraded":"2023-03-25T04:35:43.345308+0000","last_fullsized":"2023-03-25T04:35:43.345308+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:43.345324+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:35:43.345324+0000","last_peered":"2023-03-25T04:35:43.345324+0000","last_clean":"2023-03-25T04:35:43.345324+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:35:43.345324+0000","last_undegraded":"2023-03-25T04:35:43.345324+0000","last_fullsized":"2023-03-25T04:35:43.345324+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117143,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:38:54 2023","interfaces":[{"interface":"back","average":{"1min":0.25800000000000001,"5min":0.25700000000000001,"15min":0.27400000000000002},"min":{"1min":0.192,"5min":0.17399999999999999,"15min":0.16},"max":{"1min":0.39400000000000002,"5min":0.44600000000000001,"15min":0.45100000000000001},"last":0.26500000000000001},{"interface":"front","average":{"1min":0.25900000000000001,"5min":0.26700000000000002,"15min":0.27900000000000003},"min":{"1min":0.19500000000000001,"5min":0.188,"15min":0.17499999999999999},"max":{"1min":0.41499999999999998,"5min":0.46400000000000002,"15min":0.46400000000000002},"last":0.29099999999999998}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051733,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:38:45 2023","interfaces":[{"interface":"back","average":{"1min":0.27300000000000002,"5min":0.28100000000000003,"15min":0.26400000000000001},"min":{"1min":0.19400000000000001,"5min":0.17299999999999999,"15min":0.17000000000000001},"max":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"last":0.27300000000000002},{"interface":"front","average":{"1min":0.28799999999999998,"5min":0.29099999999999998,"15min":0.27800000000000002},"min":{"1min":0.19800000000000001,"5min":0.182,"15min":0.16600000000000001},"max":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"last":0.308}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:39:54.416+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:39:54.455 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:39:54.780 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:40:09.288+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:40:09.389+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:40:10.267+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)

Ignoring malparsed backtrace: 2023-03-25T04:40:28.984 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":476,"stamp":"2023-03-25T04:40:28.245911+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002693"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":39,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:38:13.358968+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:38:13.358968+0000","last_peered":"2023-03-25T04:38:13.358968+0000","last_clean":"2023-03-25T04:38:13.358968+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:38:13.358968+0000","last_undegraded":"2023-03-25T04:38:13.358968+0000","last_fullsized":"2023-03-25T04:38:13.358968+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:38:13.358922+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:38:13.358922+0000","last_peered":"2023-03-25T04:38:13.358922+0000","last_clean":"2023-03-25T04:38:13.358922+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:38:13.358922+0000","last_undegraded":"2023-03-25T04:38:13.358922+0000","last_fullsized":"2023-03-25T04:38:13.358922+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:23.353778+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:37:23.353778+0000","last_peered":"2023-03-25T04:37:23.353778+0000","last_clean":"2023-03-25T04:37:23.353778+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:37:23.353778+0000","last_undegraded":"2023-03-25T04:37:23.353778+0000","last_fullsized":"2023-03-25T04:37:23.353778+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:40.598579+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:37:40.598579+0000","last_peered":"2023-03-25T04:37:40.598579+0000","last_clean":"2023-03-25T04:37:40.598579+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:37:40.598579+0000","last_undegraded":"2023-03-25T04:37:40.598579+0000","last_fullsized":"2023-03-25T04:37:40.598579+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":39,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:48.356344+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:37:48.356344+0000","last_peered":"2023-03-25T04:37:48.356344+0000","last_clean":"2023-03-25T04:37:48.356344+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:37:48.356344+0000","last_undegraded":"2023-03-25T04:37:48.356344+0000","last_fullsized":"2023-03-25T04:37:48.356344+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:48.356365+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:37:48.356365+0000","last_peered":"2023-03-25T04:37:48.356365+0000","last_clean":"2023-03-25T04:37:48.356365+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:37:48.356365+0000","last_undegraded":"2023-03-25T04:37:48.356365+0000","last_fullsized":"2023-03-25T04:37:48.356365+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117153,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:39:54 2023","interfaces":[{"interface":"back","average":{"1min":0.25900000000000001,"5min":0.25600000000000001,"15min":0.27100000000000002},"min":{"1min":0.188,"5min":0.17899999999999999,"15min":0.16},"max":{"1min":0.41999999999999998,"5min":0.44600000000000001,"15min":0.45100000000000001},"last":0.255},{"interface":"front","average":{"1min":0.27500000000000002,"5min":0.26600000000000001,"15min":0.27700000000000002},"min":{"1min":0.17699999999999999,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.45600000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"last":0.26800000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051743,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:39:47 2023","interfaces":[{"interface":"back","average":{"1min":0.26300000000000001,"5min":0.27700000000000002,"15min":0.26500000000000001},"min":{"1min":0.187,"5min":0.17299999999999999,"15min":0.17000000000000001},"max":{"1min":0.433,"5min":0.433,"15min":0.433},"last":0.36599999999999999},{"interface":"front","average":{"1min":0.27100000000000002,"5min":0.28799999999999998,"15min":0.27900000000000003},"min":{"1min":0.20100000000000001,"5min":0.182,"15min":0.16600000000000001},"max":{"1min":0.48599999999999999,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.34999999999999998}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:40:28.986 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean
, 2023-03-25T04:40:28.986 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}

Ignoring malparsed backtrace: 2023-03-25T04:40:47.229+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:40:47.503+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:40:48.251+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)

Ignoring malparsed backtrace: 2023-03-25T04:41:03.267 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:41:03.578 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all

Ignoring malparsed backtrace: 2023-03-25T04:41:21.347 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":502,"stamp":"2023-03-25T04:41:20.256204+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002342"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":39,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:38:13.358968+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:38:13.358968+0000","last_peered":"2023-03-25T04:38:13.358968+0000","last_clean":"2023-03-25T04:38:13.358968+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:38:13.358968+0000","last_undegraded":"2023-03-25T04:38:13.358968+0000","last_fullsized":"2023-03-25T04:38:13.358968+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:38:13.358922+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:38:13.358922+0000","last_peered":"2023-03-25T04:38:13.358922+0000","last_clean":"2023-03-25T04:38:13.358922+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:38:13.358922+0000","last_undegraded":"2023-03-25T04:38:13.358922+0000","last_fullsized":"2023-03-25T04:38:13.358922+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:23.353778+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:37:23.353778+0000","last_peered":"2023-03-25T04:37:23.353778+0000","last_clean":"2023-03-25T04:37:23.353778+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:37:23.353778+0000","last_undegraded":"2023-03-25T04:37:23.353778+0000","last_fullsized":"2023-03-25T04:37:23.353778+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:40.598579+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:37:40.598579+0000","last_peered":"2023-03-25T04:37:40.598579+0000","last_clean":"2023-03-25T04:37:40.598579+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:37:40.598579+0000","last_undegraded":"2023-03-25T04:37:40.598579+0000","last_fullsized":"2023-03-25T04:37:40.598579+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":39,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:48.356344+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:37:48.356344+0000","last_peered":"2023-03-25T04:37:48.356344+0000","last_clean":"2023-03-25T04:37:48.356344+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:37:48.356344+0000","last_undegraded":"2023-03-25T04:37:48.356344+0000","last_fullsized":"2023-03-25T04:37:48.356344+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:48.356365+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:37:48.356365+0000","last_peered":"2023-03-25T04:37:48.356365+0000","last_clean":"2023-03-25T04:37:48.356365+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:37:48.356365+0000","last_undegraded":"2023-03-25T04:37:48.356365+0000","last_fullsized":"2023-03-25T04:37:48.356365+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117163,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:40:56 2023","interfaces":[{"interface":"back","average":{"1min":0.28799999999999998,"5min":0.26300000000000001,"15min":0.27100000000000002},"min":{"1min":0.188,"5min":0.184,"15min":0.16},"max":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.17499999999999999},{"interface":"front","average":{"1min":0.29599999999999999,"5min":0.27300000000000002,"15min":0.27700000000000002},"min":{"1min":0.20599999999999999,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.43099999999999999,"5min":0.46400000000000002,"15min":0.46400000000000002},"last":0.20699999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051754,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:40:52 2023","interfaces":[{"interface":"back","average":{"1min":0.28599999999999998,"5min":0.27500000000000002,"15min":0.26700000000000002},"min":{"1min":0.17899999999999999,"5min":0.17899999999999999,"15min":0.17000000000000001},"max":{"1min":0.44,"5min":0.44,"15min":0.44},"last":0.23699999999999999},{"interface":"front","average":{"1min":0.29799999999999999,"5min":0.28699999999999998,"15min":0.28100000000000003},"min":{"1min":0.20300000000000001,"5min":0.182,"15min":0.16600000000000001},"max":{"1min":0.47599999999999998,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.28000000000000003}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:41:21.349 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean
, 2023-03-25T04:41:21.349 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}

Ignoring malparsed backtrace: 2023-03-25T04:41:39.867+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:41:40.383+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:41:40.905+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:41:58.219 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json
, 2023-03-25T04:41:58.319+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:41:58.644 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:42:18.193 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":530,"stamp":"2023-03-25T04:42:16.266659+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002356"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":39,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:38:13.358968+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:38:13.358968+0000","last_peered":"2023-03-25T04:38:13.358968+0000","last_clean":"2023-03-25T04:38:13.358968+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:38:13.358968+0000","last_undegraded":"2023-03-25T04:38:13.358968+0000","last_fullsized":"2023-03-25T04:38:13.358968+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:38:13.358922+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:38:13.358922+0000","last_peered":"2023-03-25T04:38:13.358922+0000","last_clean":"2023-03-25T04:38:13.358922+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:38:13.358922+0000","last_undegraded":"2023-03-25T04:38:13.358922+0000","last_fullsized":"2023-03-25T04:38:13.358922+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:23.364593+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:39:23.364593+0000","last_peered":"2023-03-25T04:39:23.364593+0000","last_clean":"2023-03-25T04:39:23.364593+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:39:23.364593+0000","last_undegraded":"2023-03-25T04:39:23.364593+0000","last_fullsized":"2023-03-25T04:39:23.364593+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:45.608475+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:39:45.608475+0000","last_peered":"2023-03-25T04:39:45.608475+0000","last_clean":"2023-03-25T04:39:45.608475+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:39:45.608475+0000","last_undegraded":"2023-03-25T04:39:45.608475+0000","last_fullsized":"2023-03-25T04:39:45.608475+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":40,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:53.367398+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:39:53.367398+0000","last_peered":"2023-03-25T04:39:53.367398+0000","last_clean":"2023-03-25T04:39:53.367398+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:39:53.367398+0000","last_undegraded":"2023-03-25T04:39:53.367398+0000","last_fullsized":"2023-03-25T04:39:53.367398+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:53.367424+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:39:53.367424+0000","last_peered":"2023-03-25T04:39:53.367424+0000","last_clean":"2023-03-25T04:39:53.367424+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:39:53.367424+0000","last_undegraded":"2023-03-25T04:39:53.367424+0000","last_fullsized":"2023-03-25T04:39:53.367424+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117175,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:41:58 2023","interfaces":[{"interface":"back","average":{"1min":0.25800000000000001,"5min":0.26400000000000001,"15min":0.26800000000000002},"min":{"1min":0.17499999999999999,"5min":0.17499999999999999,"15min":0.16},"max":{"1min":0.39100000000000001,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.22900000000000001},{"interface":"front","average":{"1min":0.26300000000000001,"5min":0.27200000000000002,"15min":0.27400000000000002},"min":{"1min":0.20599999999999999,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.35699999999999998,"5min":0.46400000000000002,"15min":0.46400000000000002},"last":0.26400000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051765,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:41:52 2023","interfaces":[{"interface":"back","average":{"1min":0.26500000000000001,"5min":0.27200000000000002,"15min":0.26800000000000002},"min":{"1min":0.17000000000000001,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.39200000000000002,"5min":0.44,"15min":0.44},"last":0.23899999999999999},{"interface":"front","average":{"1min":0.28100000000000003,"5min":0.28599999999999998,"15min":0.28199999999999997},"min":{"1min":0.19400000000000001,"5min":0.19400000000000001,"15min":0.16600000000000001},"max":{"1min":0.41399999999999998,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.254}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:42:18.297+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:42:18.958+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:42:35.512 INFO:teuthology.orchestra.run.smithi070.stdout:
, 2023-03-25T04:42:35.512 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":1013,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":17,"modified":"2023-03-25T04:41:54.262744+0000","services":{}},"progress_events":{}}
, 2023-03-25T04:42:35.525 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:42:52.741 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":548,"stamp":"2023-03-25T04:42:52.273601+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002221"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":40,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:40:23.370056+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:40:23.370056+0000","last_peered":"2023-03-25T04:40:23.370056+0000","last_clean":"2023-03-25T04:40:23.370056+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:40:23.370056+0000","last_undegraded":"2023-03-25T04:40:23.370056+0000","last_fullsized":"2023-03-25T04:40:23.370056+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:40:23.370039+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:40:23.370039+0000","last_peered":"2023-03-25T04:40:23.370039+0000","last_clean":"2023-03-25T04:40:23.370039+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:40:23.370039+0000","last_undegraded":"2023-03-25T04:40:23.370039+0000","last_fullsized":"2023-03-25T04:40:23.370039+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:23.364593+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:39:23.364593+0000","last_peered":"2023-03-25T04:39:23.364593+0000","last_clean":"2023-03-25T04:39:23.364593+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:39:23.364593+0000","last_undegraded":"2023-03-25T04:39:23.364593+0000","last_fullsized":"2023-03-25T04:39:23.364593+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:45.608475+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:39:45.608475+0000","last_peered":"2023-03-25T04:39:45.608475+0000","last_clean":"2023-03-25T04:39:45.608475+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:39:45.608475+0000","last_undegraded":"2023-03-25T04:39:45.608475+0000","last_fullsized":"2023-03-25T04:39:45.608475+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":40,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:53.367398+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:39:53.367398+0000","last_peered":"2023-03-25T04:39:53.367398+0000","last_clean":"2023-03-25T04:39:53.367398+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:39:53.367398+0000","last_undegraded":"2023-03-25T04:39:53.367398+0000","last_fullsized":"2023-03-25T04:39:53.367398+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:53.367424+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:39:53.367424+0000","last_peered":"2023-03-25T04:39:53.367424+0000","last_clean":"2023-03-25T04:39:53.367424+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:39:53.367424+0000","last_undegraded":"2023-03-25T04:39:53.367424+0000","last_fullsized":"2023-03-25T04:39:53.367424+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117182,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:41:58 2023","interfaces":[{"interface":"back","average":{"1min":0.25800000000000001,"5min":0.26400000000000001,"15min":0.26800000000000002},"min":{"1min":0.17499999999999999,"5min":0.17499999999999999,"15min":0.16},"max":{"1min":0.39100000000000001,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.19600000000000001},{"interface":"front","average":{"1min":0.26300000000000001,"5min":0.27200000000000002,"15min":0.27400000000000002},"min":{"1min":0.20599999999999999,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.35699999999999998,"5min":0.46400000000000002,"15min":0.46400000000000002},"last":0.219}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051773,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:41:52 2023","interfaces":[{"interface":"back","average":{"1min":0.26500000000000001,"5min":0.27200000000000002,"15min":0.26800000000000002},"min":{"1min":0.17000000000000001,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.39200000000000002,"5min":0.44,"15min":0.44},"last":0.19},{"interface":"front","average":{"1min":0.28100000000000003,"5min":0.28599999999999998,"15min":0.28199999999999997},"min":{"1min":0.19400000000000001,"5min":0.19400000000000001,"15min":0.16600000000000001},"max":{"1min":0.41399999999999998,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.17599999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:42:52.742 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean

Ignoring malparsed backtrace: 2023-03-25T04:43:09.973 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:43:09.986 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":556,"stamp":"2023-03-25T04:43:08.276556+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002359"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":40,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:40:23.370056+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:40:23.370056+0000","last_peered":"2023-03-25T04:40:23.370056+0000","last_clean":"2023-03-25T04:40:23.370056+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:40:23.370056+0000","last_undegraded":"2023-03-25T04:40:23.370056+0000","last_fullsized":"2023-03-25T04:40:23.370056+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:40:23.370039+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:40:23.370039+0000","last_peered":"2023-03-25T04:40:23.370039+0000","last_clean":"2023-03-25T04:40:23.370039+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:40:23.370039+0000","last_undegraded":"2023-03-25T04:40:23.370039+0000","last_fullsized":"2023-03-25T04:40:23.370039+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:23.364593+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:39:23.364593+0000","last_peered":"2023-03-25T04:39:23.364593+0000","last_clean":"2023-03-25T04:39:23.364593+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:39:23.364593+0000","last_undegraded":"2023-03-25T04:39:23.364593+0000","last_fullsized":"2023-03-25T04:39:23.364593+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:45.608475+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:39:45.608475+0000","last_peered":"2023-03-25T04:39:45.608475+0000","last_clean":"2023-03-25T04:39:45.608475+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:39:45.608475+0000","last_undegraded":"2023-03-25T04:39:45.608475+0000","last_fullsized":"2023-03-25T04:39:45.608475+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":40,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:53.367398+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:39:53.367398+0000","last_peered":"2023-03-25T04:39:53.367398+0000","last_clean":"2023-03-25T04:39:53.367398+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:39:53.367398+0000","last_undegraded":"2023-03-25T04:39:53.367398+0000","last_fullsized":"2023-03-25T04:39:53.367398+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:53.367424+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:39:53.367424+0000","last_peered":"2023-03-25T04:39:53.367424+0000","last_clean":"2023-03-25T04:39:53.367424+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:39:53.367424+0000","last_undegraded":"2023-03-25T04:39:53.367424+0000","last_fullsized":"2023-03-25T04:39:53.367424+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117185,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:43:00 2023","interfaces":[{"interface":"back","average":{"1min":0.27400000000000002,"5min":0.26700000000000002,"15min":0.26600000000000001},"min":{"1min":0.19600000000000001,"5min":0.17499999999999999,"15min":0.16},"max":{"1min":0.42999999999999999,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.27800000000000002},{"interface":"front","average":{"1min":0.28199999999999997,"5min":0.27500000000000002,"15min":0.27300000000000002},"min":{"1min":0.185,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.47199999999999998,"5min":0.47199999999999998,"15min":0.47199999999999998},"last":0.247}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051776,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:42:54 2023","interfaces":[{"interface":"back","average":{"1min":0.26300000000000001,"5min":0.27000000000000002,"15min":0.26900000000000002},"min":{"1min":0.17399999999999999,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.438,"5min":0.44,"15min":0.44},"last":0.20100000000000001},{"interface":"front","average":{"1min":0.27400000000000002,"5min":0.28199999999999997,"15min":0.28299999999999997},"min":{"1min":0.17599999999999999,"5min":0.17599999999999999,"15min":0.16600000000000001},"max":{"1min":0.41599999999999998,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.23499999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:43:25.042+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:43:25.411+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:43:26.013+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:43:47.937+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:43:48.534+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:43:48.889+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:44:09.318 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:44:09.331 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":586,"stamp":"2023-03-25T04:44:08.288933+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002697"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":40,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:40:23.370056+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:40:23.370056+0000","last_peered":"2023-03-25T04:40:23.370056+0000","last_clean":"2023-03-25T04:40:23.370056+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:40:23.370056+0000","last_undegraded":"2023-03-25T04:40:23.370056+0000","last_fullsized":"2023-03-25T04:40:23.370056+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:40:23.370039+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:40:23.370039+0000","last_peered":"2023-03-25T04:40:23.370039+0000","last_clean":"2023-03-25T04:40:23.370039+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:40:23.370039+0000","last_undegraded":"2023-03-25T04:40:23.370039+0000","last_fullsized":"2023-03-25T04:40:23.370039+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:23.375106+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:41:23.375106+0000","last_peered":"2023-03-25T04:41:23.375106+0000","last_clean":"2023-03-25T04:41:23.375106+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:41:23.375106+0000","last_undegraded":"2023-03-25T04:41:23.375106+0000","last_fullsized":"2023-03-25T04:41:23.375106+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:50.618930+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:41:50.618930+0000","last_peered":"2023-03-25T04:41:50.618930+0000","last_clean":"2023-03-25T04:41:50.618930+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:41:50.618930+0000","last_undegraded":"2023-03-25T04:41:50.618930+0000","last_fullsized":"2023-03-25T04:41:50.618930+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":41,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:58.377821+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:41:58.377821+0000","last_peered":"2023-03-25T04:41:58.377821+0000","last_clean":"2023-03-25T04:41:58.377821+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:41:58.377821+0000","last_undegraded":"2023-03-25T04:41:58.377821+0000","last_fullsized":"2023-03-25T04:41:58.377821+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:58.377836+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:41:58.377836+0000","last_peered":"2023-03-25T04:41:58.377836+0000","last_clean":"2023-03-25T04:41:58.377836+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:41:58.377836+0000","last_undegraded":"2023-03-25T04:41:58.377836+0000","last_fullsized":"2023-03-25T04:41:58.377836+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117197,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:44:01 2023","interfaces":[{"interface":"back","average":{"1min":0.27200000000000002,"5min":0.27000000000000002,"15min":0.26400000000000001},"min":{"1min":0.20399999999999999,"5min":0.17499999999999999,"15min":0.16},"max":{"1min":0.38400000000000001,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.245},{"interface":"front","average":{"1min":0.27600000000000002,"5min":0.27800000000000002,"15min":0.27100000000000002},"min":{"1min":0.23100000000000001,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.40400000000000003,"5min":0.47199999999999998,"15min":0.47199999999999998},"last":0.27500000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051788,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:43:55 2023","interfaces":[{"interface":"back","average":{"1min":0.26100000000000001,"5min":0.26700000000000002,"15min":0.27000000000000002},"min":{"1min":0.17799999999999999,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.374,"5min":0.44,"15min":0.44},"last":0.30299999999999999},{"interface":"front","average":{"1min":0.28299999999999997,"5min":0.28100000000000003,"15min":0.28399999999999997},"min":{"1min":0.20899999999999999,"5min":0.17599999999999999,"15min":0.16600000000000001},"max":{"1min":0.40100000000000002,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.27800000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:44:28.923 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":1126,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":19,"modified":"2023-03-25T04:43:20.278959+0000","services":{}},"progress_events":{}}
, 2023-03-25T04:44:28.936 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now
, 2023-03-25T04:44:28.937 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace: 2023-03-25T04:44:48.786 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:44:48.800 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":606,"stamp":"2023-03-25T04:44:48.296805+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002182"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":41,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:42:34.382196+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:42:34.382196+0000","last_peered":"2023-03-25T04:42:34.382196+0000","last_clean":"2023-03-25T04:42:34.382196+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:42:34.382196+0000","last_undegraded":"2023-03-25T04:42:34.382196+0000","last_fullsized":"2023-03-25T04:42:34.382196+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:42:29.381744+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:42:29.381744+0000","last_peered":"2023-03-25T04:42:29.381744+0000","last_clean":"2023-03-25T04:42:29.381744+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:42:29.381744+0000","last_undegraded":"2023-03-25T04:42:29.381744+0000","last_fullsized":"2023-03-25T04:42:29.381744+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:23.375106+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:41:23.375106+0000","last_peered":"2023-03-25T04:41:23.375106+0000","last_clean":"2023-03-25T04:41:23.375106+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:41:23.375106+0000","last_undegraded":"2023-03-25T04:41:23.375106+0000","last_fullsized":"2023-03-25T04:41:23.375106+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:50.618930+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:41:50.618930+0000","last_peered":"2023-03-25T04:41:50.618930+0000","last_clean":"2023-03-25T04:41:50.618930+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:41:50.618930+0000","last_undegraded":"2023-03-25T04:41:50.618930+0000","last_fullsized":"2023-03-25T04:41:50.618930+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":41,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:58.377821+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:41:58.377821+0000","last_peered":"2023-03-25T04:41:58.377821+0000","last_clean":"2023-03-25T04:41:58.377821+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:41:58.377821+0000","last_undegraded":"2023-03-25T04:41:58.377821+0000","last_fullsized":"2023-03-25T04:41:58.377821+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:58.377836+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:41:58.377836+0000","last_peered":"2023-03-25T04:41:58.377836+0000","last_clean":"2023-03-25T04:41:58.377836+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:41:58.377836+0000","last_undegraded":"2023-03-25T04:41:58.377836+0000","last_fullsized":"2023-03-25T04:41:58.377836+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117206,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:44:01 2023","interfaces":[{"interface":"back","average":{"1min":0.27200000000000002,"5min":0.27000000000000002,"15min":0.26400000000000001},"min":{"1min":0.20399999999999999,"5min":0.17499999999999999,"15min":0.16},"max":{"1min":0.38400000000000001,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.216},{"interface":"front","average":{"1min":0.27600000000000002,"5min":0.27800000000000002,"15min":0.27100000000000002},"min":{"1min":0.23100000000000001,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.40400000000000003,"5min":0.47199999999999998,"15min":0.47199999999999998},"last":0.23999999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051796,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:43:55 2023","interfaces":[{"interface":"back","average":{"1min":0.26100000000000001,"5min":0.26700000000000002,"15min":0.27000000000000002},"min":{"1min":0.17799999999999999,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.374,"5min":0.44,"15min":0.44},"last":0.25600000000000001},{"interface":"front","average":{"1min":0.28299999999999997,"5min":0.28100000000000003,"15min":0.28399999999999997},"min":{"1min":0.20899999999999999,"5min":0.17599999999999999,"15min":0.16600000000000001},"max":{"1min":0.40100000000000002,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.28000000000000003}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:45:05.762 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:45:06.073 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all

Ignoring malparsed backtrace: 2023-03-25T04:45:25.658 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":624,"stamp":"2023-03-25T04:45:24.304294+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002366"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":41,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:42:34.382196+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:42:34.382196+0000","last_peered":"2023-03-25T04:42:34.382196+0000","last_clean":"2023-03-25T04:42:34.382196+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:42:34.382196+0000","last_undegraded":"2023-03-25T04:42:34.382196+0000","last_fullsized":"2023-03-25T04:42:34.382196+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:42:29.381744+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:42:29.381744+0000","last_peered":"2023-03-25T04:42:29.381744+0000","last_clean":"2023-03-25T04:42:29.381744+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:42:29.381744+0000","last_undegraded":"2023-03-25T04:42:29.381744+0000","last_fullsized":"2023-03-25T04:42:29.381744+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:23.375106+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:41:23.375106+0000","last_peered":"2023-03-25T04:41:23.375106+0000","last_clean":"2023-03-25T04:41:23.375106+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:41:23.375106+0000","last_undegraded":"2023-03-25T04:41:23.375106+0000","last_fullsized":"2023-03-25T04:41:23.375106+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:50.618930+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:41:50.618930+0000","last_peered":"2023-03-25T04:41:50.618930+0000","last_clean":"2023-03-25T04:41:50.618930+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:41:50.618930+0000","last_undegraded":"2023-03-25T04:41:50.618930+0000","last_fullsized":"2023-03-25T04:41:50.618930+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":41,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:58.377821+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:41:58.377821+0000","last_peered":"2023-03-25T04:41:58.377821+0000","last_clean":"2023-03-25T04:41:58.377821+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:41:58.377821+0000","last_undegraded":"2023-03-25T04:41:58.377821+0000","last_fullsized":"2023-03-25T04:41:58.377821+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:58.377836+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:41:58.377836+0000","last_peered":"2023-03-25T04:41:58.377836+0000","last_clean":"2023-03-25T04:41:58.377836+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:41:58.377836+0000","last_undegraded":"2023-03-25T04:41:58.377836+0000","last_fullsized":"2023-03-25T04:41:58.377836+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117213,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:45:01 2023","interfaces":[{"interface":"back","average":{"1min":0.251,"5min":0.26800000000000002,"15min":0.26400000000000001},"min":{"1min":0.183,"5min":0.17499999999999999,"15min":0.16},"max":{"1min":0.41499999999999998,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.26900000000000002},{"interface":"front","average":{"1min":0.28000000000000003,"5min":0.27900000000000003,"15min":0.27100000000000002},"min":{"1min":0.20999999999999999,"5min":0.185,"15min":0.17499999999999999},"max":{"1min":0.44400000000000001,"5min":0.47199999999999998,"15min":0.47199999999999998},"last":0.27900000000000003}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051803,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:44:55 2023","interfaces":[{"interface":"back","average":{"1min":0.252,"5min":0.26500000000000001,"15min":0.26900000000000002},"min":{"1min":0.183,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.39700000000000002,"5min":0.44,"15min":0.44},"last":0.20699999999999999},{"interface":"front","average":{"1min":0.27000000000000002,"5min":0.28100000000000003,"15min":0.28299999999999997},"min":{"1min":0.20300000000000001,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":0.42099999999999999,"5min":0.47599999999999998,"15min":0.48599999999999999},"last":0.23599999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:45:25.659 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:45:25.969 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:45:42.735+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:45:42.744+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:45:42.897 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace: 2023-03-25T04:46:00.183 INFO:teuthology.orchestra.run.smithi070.stdout:
, 2023-03-25T04:46:00.183 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":1217,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":20,"modified":"2023-03-25T04:45:42.307700+0000","services":{"mgr":{"daemons":{"summary":"","x":{"start_epoch":0,"start_stamp":"0.000000","gid":0,"addr":"(unrecognized address family 0)/0","metadata":{},"task_status":{}}}}}},"progress_events":{}}
, 2023-03-25T04:46:00.197 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now

Ignoring malparsed backtrace: 2023-03-25T04:46:19.910+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:46:20.039 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all

Ignoring malparsed backtrace: 2023-03-25T04:46:39.611 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}
, 2023-03-25T04:46:39.611 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json
, 2023-03-25T04:46:39.613+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:46:58.485+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:46:58.764+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:46:59.519+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:47:14.132 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:47:14.146 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":678,"stamp":"2023-03-25T04:47:12.325044+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002571"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":42,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:44.393426+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:44:44.393426+0000","last_peered":"2023-03-25T04:44:44.393426+0000","last_clean":"2023-03-25T04:44:44.393426+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:44:44.393426+0000","last_undegraded":"2023-03-25T04:44:44.393426+0000","last_fullsized":"2023-03-25T04:44:44.393426+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:39.393044+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:44:39.393044+0000","last_peered":"2023-03-25T04:44:39.393044+0000","last_clean":"2023-03-25T04:44:39.393044+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:44:39.393044+0000","last_undegraded":"2023-03-25T04:44:39.393044+0000","last_fullsized":"2023-03-25T04:44:39.393044+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:43:24.386177+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:43:24.386177+0000","last_peered":"2023-03-25T04:43:24.386177+0000","last_clean":"2023-03-25T04:43:24.386177+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:43:24.386177+0000","last_undegraded":"2023-03-25T04:43:24.386177+0000","last_fullsized":"2023-03-25T04:43:24.386177+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:43:55.629798+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:43:55.629798+0000","last_peered":"2023-03-25T04:43:55.629798+0000","last_clean":"2023-03-25T04:43:55.629798+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:43:55.629798+0000","last_undegraded":"2023-03-25T04:43:55.629798+0000","last_fullsized":"2023-03-25T04:43:55.629798+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":42,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:04.389904+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:44:04.389904+0000","last_peered":"2023-03-25T04:44:04.389904+0000","last_clean":"2023-03-25T04:44:04.389904+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:44:04.389904+0000","last_undegraded":"2023-03-25T04:44:04.389904+0000","last_fullsized":"2023-03-25T04:44:04.389904+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:04.389923+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:44:04.389923+0000","last_peered":"2023-03-25T04:44:04.389923+0000","last_clean":"2023-03-25T04:44:04.389923+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:44:04.389923+0000","last_undegraded":"2023-03-25T04:44:04.389923+0000","last_fullsized":"2023-03-25T04:44:04.389923+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117235,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:47:05 2023","interfaces":[{"interface":"back","average":{"1min":0.27800000000000002,"5min":0.27000000000000002,"15min":0.26300000000000001},"min":{"1min":0.20100000000000001,"5min":0.16900000000000001,"15min":0.16},"max":{"1min":0.39200000000000002,"5min":0.42999999999999999,"15min":0.45600000000000002},"last":0.379},{"interface":"front","average":{"1min":0.28399999999999997,"5min":0.28100000000000003,"15min":0.27200000000000002},"min":{"1min":0.17799999999999999,"5min":0.17799999999999999,"15min":0.17699999999999999},"max":{"1min":0.42599999999999999,"5min":0.47199999999999998,"15min":0.47199999999999998},"last":0.39800000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051825,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:47:01 2023","interfaces":[{"interface":"back","average":{"1min":0.27300000000000002,"5min":0.26100000000000001,"15min":0.27000000000000002},"min":{"1min":0.186,"5min":0.17399999999999999,"15min":0.17000000000000001},"max":{"1min":0.40400000000000003,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.23899999999999999},{"interface":"front","average":{"1min":0.28699999999999998,"5min":0.27500000000000002,"15min":0.28299999999999997},"min":{"1min":0.186,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.48599999999999999},"last":0.19600000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:47:31.198 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json
, 2023-03-25T04:47:31.411+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:47:31.789 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:47:49.509 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":696,"stamp":"2023-03-25T04:47:48.332093+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002697"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":42,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:44.393426+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:44:44.393426+0000","last_peered":"2023-03-25T04:44:44.393426+0000","last_clean":"2023-03-25T04:44:44.393426+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:44:44.393426+0000","last_undegraded":"2023-03-25T04:44:44.393426+0000","last_fullsized":"2023-03-25T04:44:44.393426+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:39.393044+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:44:39.393044+0000","last_peered":"2023-03-25T04:44:39.393044+0000","last_clean":"2023-03-25T04:44:39.393044+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:44:39.393044+0000","last_undegraded":"2023-03-25T04:44:39.393044+0000","last_fullsized":"2023-03-25T04:44:39.393044+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:45:24.396717+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:45:24.396717+0000","last_peered":"2023-03-25T04:45:24.396717+0000","last_clean":"2023-03-25T04:45:24.396717+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:45:24.396717+0000","last_undegraded":"2023-03-25T04:45:24.396717+0000","last_fullsized":"2023-03-25T04:45:24.396717+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:43:55.629798+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:43:55.629798+0000","last_peered":"2023-03-25T04:43:55.629798+0000","last_clean":"2023-03-25T04:43:55.629798+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:43:55.629798+0000","last_undegraded":"2023-03-25T04:43:55.629798+0000","last_fullsized":"2023-03-25T04:43:55.629798+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":42,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:04.389904+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:44:04.389904+0000","last_peered":"2023-03-25T04:44:04.389904+0000","last_clean":"2023-03-25T04:44:04.389904+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:44:04.389904+0000","last_undegraded":"2023-03-25T04:44:04.389904+0000","last_fullsized":"2023-03-25T04:44:04.389904+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:04.389923+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:44:04.389923+0000","last_peered":"2023-03-25T04:44:04.389923+0000","last_clean":"2023-03-25T04:44:04.389923+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:44:04.389923+0000","last_undegraded":"2023-03-25T04:44:04.389923+0000","last_fullsized":"2023-03-25T04:44:04.389923+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117242,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:47:05 2023","interfaces":[{"interface":"back","average":{"1min":0.27800000000000002,"5min":0.27000000000000002,"15min":0.26300000000000001},"min":{"1min":0.20100000000000001,"5min":0.16900000000000001,"15min":0.16},"max":{"1min":0.39200000000000002,"5min":0.42999999999999999,"15min":0.45600000000000002},"last":0.40500000000000003},{"interface":"front","average":{"1min":0.28399999999999997,"5min":0.28100000000000003,"15min":0.27200000000000002},"min":{"1min":0.17799999999999999,"5min":0.17799999999999999,"15min":0.17699999999999999},"max":{"1min":0.42599999999999999,"5min":0.47199999999999998,"15min":0.47199999999999998},"last":0.42399999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051834,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:47:01 2023","interfaces":[{"interface":"back","average":{"1min":0.27300000000000002,"5min":0.26100000000000001,"15min":0.27000000000000002},"min":{"1min":0.186,"5min":0.17399999999999999,"15min":0.17000000000000001},"max":{"1min":0.40400000000000003,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.25600000000000001},{"interface":"front","average":{"1min":0.28699999999999998,"5min":0.27500000000000002,"15min":0.28299999999999997},"min":{"1min":0.186,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.48599999999999999},"last":0.22800000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:47:49.510 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace:     ctx.managers[config['cluster']].wait_for_clean()
,   File "/home/teuthworker/src/github.com_ceph_ceph-c_511015c23eef0684a64171b710e432448df2b7c7/qa/tasks/ceph_manager.py", line 2747, in wait_for_clean
,     assert time.time() - start < timeout, \

Ignoring malparsed backtrace:  ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
,  1: /lib64/libpthread.so.0(+0x12cf0) [0x7f1453c2fcf0]
,  2: __pthread_mutex_lock()

Ignoring malparsed backtrace: 2023-03-25T04:28:05.172 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":142,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":5,"modified":"2023-03-25T04:27:12.092098+0000","services":{}},"progress_events":{}}
, 2023-03-25T04:28:05.185 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now
, 2023-03-25T04:28:05.186 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace: 2023-03-25T04:28:28.360 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":116,"stamp":"2023-03-25T04:28:28.108007+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002578"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.314661+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:27:45.314661+0000","last_peered":"2023-03-25T04:27:45.314661+0000","last_clean":"2023-03-25T04:27:45.314661+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:27:45.314661+0000","last_undegraded":"2023-03-25T04:27:45.314661+0000","last_fullsized":"2023-03-25T04:27:45.314661+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":25,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.310745+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:27:44.310745+0000","last_peered":"2023-03-25T04:27:44.310745+0000","last_clean":"2023-03-25T04:27:44.310745+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:27:44.310745+0000","last_undegraded":"2023-03-25T04:27:44.310745+0000","last_fullsized":"2023-03-25T04:27:44.310745+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":25,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.310650+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:27:44.310650+0000","last_peered":"2023-03-25T04:27:44.310650+0000","last_clean":"2023-03-25T04:27:44.310650+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:27:44.310650+0000","last_undegraded":"2023-03-25T04:27:44.310650+0000","last_fullsized":"2023-03-25T04:27:44.310650+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":26,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.312962+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:27:44.312962+0000","last_peered":"2023-03-25T04:27:44.312962+0000","last_clean":"2023-03-25T04:27:44.312962+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:27:44.312962+0000","last_undegraded":"2023-03-25T04:27:44.312962+0000","last_fullsized":"2023-03-25T04:27:44.312962+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315064+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:27:45.315064+0000","last_peered":"2023-03-25T04:27:45.315064+0000","last_clean":"2023-03-25T04:27:45.315064+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:27:45.315064+0000","last_undegraded":"2023-03-25T04:27:45.315064+0000","last_fullsized":"2023-03-25T04:27:45.315064+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":25,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.310707+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:27:44.310707+0000","last_peered":"2023-03-25T04:27:44.310707+0000","last_clean":"2023-03-25T04:27:44.310707+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:27:44.310707+0000","last_undegraded":"2023-03-25T04:27:44.310707+0000","last_fullsized":"2023-03-25T04:27:44.310707+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117007,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.375}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051597,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.186}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:28:28.362 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean
, 2023-03-25T04:28:28.362 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:28:45.655 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":124,"stamp":"2023-03-25T04:28:44.111265+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002492"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.314661+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:27:45.314661+0000","last_peered":"2023-03-25T04:27:45.314661+0000","last_clean":"2023-03-25T04:27:45.314661+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:27:45.314661+0000","last_undegraded":"2023-03-25T04:27:45.314661+0000","last_fullsized":"2023-03-25T04:27:45.314661+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":25,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.310745+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:27:44.310745+0000","last_peered":"2023-03-25T04:27:44.310745+0000","last_clean":"2023-03-25T04:27:44.310745+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:27:44.310745+0000","last_undegraded":"2023-03-25T04:27:44.310745+0000","last_fullsized":"2023-03-25T04:27:44.310745+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":25,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.310650+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:27:44.310650+0000","last_peered":"2023-03-25T04:27:44.310650+0000","last_clean":"2023-03-25T04:27:44.310650+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:27:44.310650+0000","last_undegraded":"2023-03-25T04:27:44.310650+0000","last_fullsized":"2023-03-25T04:27:44.310650+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":26,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.312962+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:27:44.312962+0000","last_peered":"2023-03-25T04:27:44.312962+0000","last_clean":"2023-03-25T04:27:44.312962+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:27:44.312962+0000","last_undegraded":"2023-03-25T04:27:44.312962+0000","last_fullsized":"2023-03-25T04:27:44.312962+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315064+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:27:45.315064+0000","last_peered":"2023-03-25T04:27:45.315064+0000","last_clean":"2023-03-25T04:27:45.315064+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:27:45.315064+0000","last_undegraded":"2023-03-25T04:27:45.315064+0000","last_fullsized":"2023-03-25T04:27:45.315064+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":25,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.310707+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:27:44.310707+0000","last_peered":"2023-03-25T04:27:44.310707+0000","last_clean":"2023-03-25T04:27:44.310707+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:27:44.310707+0000","last_undegraded":"2023-03-25T04:27:44.310707+0000","last_fullsized":"2023-03-25T04:27:44.310707+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117011,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:28:31 2023","interfaces":[{"interface":"back","average":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.36899999999999999},{"interface":"front","average":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"min":{"1min":0.19,"5min":0.19,"15min":0.19},"max":{"1min":0.46300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.39800000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051601,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:28:24 2023","interfaces":[{"interface":"back","average":{"1min":0.249,"5min":0.249,"15min":0.249},"min":{"1min":0.17699999999999999,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"last":0.254},{"interface":"front","average":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"min":{"1min":0.20200000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"last":0.312}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:28:45.656 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace: 2023-03-25T04:29:05.171 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}
, 2023-03-25T04:29:05.171 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json
, 2023-03-25T04:29:05.561 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:29:22.465 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:29:22.773 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all

Ignoring malparsed backtrace: 2023-03-25T04:29:42.405 INFO:teuthology.orchestra.run.smithi070.stdout:
, 2023-03-25T04:29:42.406 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":240,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":9,"modified":"2023-03-25T04:29:32.120358+0000","services":{"mon":{"daemons":{"summary":"","a":{"start_epoch":0,"start_stamp":"0.000000","gid":0,"addr":"(unrecognized address family 0)/0","metadata":{},"task_status":{}}}}}},"progress_events":{}}
, 2023-03-25T04:29:42.419 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:29:59.627 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":161,"stamp":"2023-03-25T04:29:58.125186+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002398"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315468+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:27:45.315468+0000","last_peered":"2023-03-25T04:27:45.315468+0000","last_clean":"2023-03-25T04:27:45.315468+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:27:45.315468+0000","last_undegraded":"2023-03-25T04:27:45.315468+0000","last_fullsized":"2023-03-25T04:27:45.315468+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312845+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:27:45.312845+0000","last_peered":"2023-03-25T04:27:45.312845+0000","last_clean":"2023-03-25T04:27:45.312845+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:27:45.312845+0000","last_undegraded":"2023-03-25T04:27:45.312845+0000","last_fullsized":"2023-03-25T04:27:45.312845+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312882+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:27:45.312882+0000","last_peered":"2023-03-25T04:27:45.312882+0000","last_clean":"2023-03-25T04:27:45.312882+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:27:45.312882+0000","last_undegraded":"2023-03-25T04:27:45.312882+0000","last_fullsized":"2023-03-25T04:27:45.312882+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.311730+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:27:45.311730+0000","last_peered":"2023-03-25T04:27:45.311730+0000","last_clean":"2023-03-25T04:27:45.311730+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:27:45.311730+0000","last_undegraded":"2023-03-25T04:27:45.311730+0000","last_fullsized":"2023-03-25T04:27:45.311730+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315735+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:27:45.315735+0000","last_peered":"2023-03-25T04:27:45.315735+0000","last_clean":"2023-03-25T04:27:45.315735+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:27:45.315735+0000","last_undegraded":"2023-03-25T04:27:45.315735+0000","last_fullsized":"2023-03-25T04:27:45.315735+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312833+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:27:45.312833+0000","last_peered":"2023-03-25T04:27:45.312833+0000","last_clean":"2023-03-25T04:27:45.312833+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:27:45.312833+0000","last_undegraded":"2023-03-25T04:27:45.312833+0000","last_fullsized":"2023-03-25T04:27:45.312833+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117026,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:29:33 2023","interfaces":[{"interface":"back","average":{"1min":0.26300000000000001,"5min":0.29099999999999998,"15min":0.29599999999999999},"min":{"1min":0.191,"5min":0.191,"15min":0.191},"max":{"1min":0.371,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.20499999999999999},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.29599999999999999,"15min":0.29999999999999999},"min":{"1min":0.21099999999999999,"5min":0.19,"15min":0.19},"max":{"1min":0.39800000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.19500000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051616,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:29:29 2023","interfaces":[{"interface":"back","average":{"1min":0.27200000000000002,"5min":0.253,"15min":0.25},"min":{"1min":0.183,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"last":0.26400000000000001},{"interface":"front","average":{"1min":0.28499999999999998,"5min":0.26800000000000002,"15min":0.26500000000000001},"min":{"1min":0.16600000000000001,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"last":0.29899999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:29:59.629 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean

Ignoring malparsed backtrace: 2023-03-25T04:30:16.889 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:30:16.903 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":170,"stamp":"2023-03-25T04:30:16.128427+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002222"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315468+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:27:45.315468+0000","last_peered":"2023-03-25T04:27:45.315468+0000","last_clean":"2023-03-25T04:27:45.315468+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:27:45.315468+0000","last_undegraded":"2023-03-25T04:27:45.315468+0000","last_fullsized":"2023-03-25T04:27:45.315468+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312845+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:27:45.312845+0000","last_peered":"2023-03-25T04:27:45.312845+0000","last_clean":"2023-03-25T04:27:45.312845+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:27:45.312845+0000","last_undegraded":"2023-03-25T04:27:45.312845+0000","last_fullsized":"2023-03-25T04:27:45.312845+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312882+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:27:45.312882+0000","last_peered":"2023-03-25T04:27:45.312882+0000","last_clean":"2023-03-25T04:27:45.312882+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:27:45.312882+0000","last_undegraded":"2023-03-25T04:27:45.312882+0000","last_fullsized":"2023-03-25T04:27:45.312882+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.311730+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:27:45.311730+0000","last_peered":"2023-03-25T04:27:45.311730+0000","last_clean":"2023-03-25T04:27:45.311730+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:27:45.311730+0000","last_undegraded":"2023-03-25T04:27:45.311730+0000","last_fullsized":"2023-03-25T04:27:45.311730+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315735+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:27:45.315735+0000","last_peered":"2023-03-25T04:27:45.315735+0000","last_clean":"2023-03-25T04:27:45.315735+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:27:45.315735+0000","last_undegraded":"2023-03-25T04:27:45.315735+0000","last_fullsized":"2023-03-25T04:27:45.315735+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312833+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:27:45.312833+0000","last_peered":"2023-03-25T04:27:45.312833+0000","last_clean":"2023-03-25T04:27:45.312833+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:27:45.312833+0000","last_undegraded":"2023-03-25T04:27:45.312833+0000","last_fullsized":"2023-03-25T04:27:45.312833+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117030,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:29:33 2023","interfaces":[{"interface":"back","average":{"1min":0.26300000000000001,"5min":0.29099999999999998,"15min":0.29599999999999999},"min":{"1min":0.191,"5min":0.191,"15min":0.191},"max":{"1min":0.371,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.245},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.29599999999999999,"15min":0.29999999999999999},"min":{"1min":0.21099999999999999,"5min":0.19,"15min":0.19},"max":{"1min":0.39800000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.20899999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051620,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:29:29 2023","interfaces":[{"interface":"back","average":{"1min":0.27200000000000002,"5min":0.253,"15min":0.25},"min":{"1min":0.183,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"last":0.19400000000000001},{"interface":"front","average":{"1min":0.28499999999999998,"5min":0.26800000000000002,"15min":0.26500000000000001},"min":{"1min":0.16600000000000001,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"last":0.23599999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:30:31.984+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:30:32.201+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:30:32.981+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)

Ignoring malparsed backtrace: 2023-03-25T04:30:48.449 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now
, 2023-03-25T04:30:48.450 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:30:48.760 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:31:08.342 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":196,"stamp":"2023-03-25T04:31:08.139018+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002431"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315468+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:27:45.315468+0000","last_peered":"2023-03-25T04:27:45.315468+0000","last_clean":"2023-03-25T04:27:45.315468+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:27:45.315468+0000","last_undegraded":"2023-03-25T04:27:45.315468+0000","last_fullsized":"2023-03-25T04:27:45.315468+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312845+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:27:45.312845+0000","last_peered":"2023-03-25T04:27:45.312845+0000","last_clean":"2023-03-25T04:27:45.312845+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:27:45.312845+0000","last_undegraded":"2023-03-25T04:27:45.312845+0000","last_fullsized":"2023-03-25T04:27:45.312845+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312882+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:27:45.312882+0000","last_peered":"2023-03-25T04:27:45.312882+0000","last_clean":"2023-03-25T04:27:45.312882+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:27:45.312882+0000","last_undegraded":"2023-03-25T04:27:45.312882+0000","last_fullsized":"2023-03-25T04:27:45.312882+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.311730+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:27:45.311730+0000","last_peered":"2023-03-25T04:27:45.311730+0000","last_clean":"2023-03-25T04:27:45.311730+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:27:45.311730+0000","last_undegraded":"2023-03-25T04:27:45.311730+0000","last_fullsized":"2023-03-25T04:27:45.311730+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315735+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:27:45.315735+0000","last_peered":"2023-03-25T04:27:45.315735+0000","last_clean":"2023-03-25T04:27:45.315735+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:27:45.315735+0000","last_undegraded":"2023-03-25T04:27:45.315735+0000","last_fullsized":"2023-03-25T04:27:45.315735+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312833+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:27:45.312833+0000","last_peered":"2023-03-25T04:27:45.312833+0000","last_clean":"2023-03-25T04:27:45.312833+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:27:45.312833+0000","last_undegraded":"2023-03-25T04:27:45.312833+0000","last_fullsized":"2023-03-25T04:27:45.312833+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117040,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:30:37 2023","interfaces":[{"interface":"back","average":{"1min":0.28799999999999998,"5min":0.28899999999999998,"15min":0.29499999999999998},"min":{"1min":0.20200000000000001,"5min":0.191,"15min":0.191},"max":{"1min":0.433,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.29799999999999999},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.28999999999999998,"15min":0.29799999999999999},"min":{"1min":0.17499999999999999,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":0.40300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.25700000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051630,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:30:32 2023","interfaces":[{"interface":"back","average":{"1min":0.25600000000000001,"5min":0.255,"15min":0.251},"min":{"1min":0.19400000000000001,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.35699999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"last":0.22800000000000001},{"interface":"front","average":{"1min":0.28699999999999998,"5min":0.27200000000000002,"15min":0.26600000000000001},"min":{"1min":0.20599999999999999,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.39400000000000002,"5min":0.42899999999999999,"15min":0.42899999999999999},"last":0.24099999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:31:08.343 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace: 2023-03-25T04:31:24.283+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:31:25.192+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:31:25.295+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)

Ignoring malparsed backtrace: 2023-03-25T04:31:40.257 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:31:40.271 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":212,"stamp":"2023-03-25T04:31:40.144732+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002208"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315468+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:27:45.315468+0000","last_peered":"2023-03-25T04:27:45.315468+0000","last_clean":"2023-03-25T04:27:45.315468+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:27:45.315468+0000","last_undegraded":"2023-03-25T04:27:45.315468+0000","last_fullsized":"2023-03-25T04:27:45.315468+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312845+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:27:45.312845+0000","last_peered":"2023-03-25T04:27:45.312845+0000","last_clean":"2023-03-25T04:27:45.312845+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:27:45.312845+0000","last_undegraded":"2023-03-25T04:27:45.312845+0000","last_fullsized":"2023-03-25T04:27:45.312845+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:23.308607+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:29:23.308607+0000","last_peered":"2023-03-25T04:29:23.308607+0000","last_clean":"2023-03-25T04:29:23.308607+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:29:23.308607+0000","last_undegraded":"2023-03-25T04:29:23.308607+0000","last_fullsized":"2023-03-25T04:29:23.308607+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":28,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:20.554710+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:29:20.554710+0000","last_peered":"2023-03-25T04:29:20.554710+0000","last_clean":"2023-03-25T04:29:20.554710+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:29:20.554710+0000","last_undegraded":"2023-03-25T04:29:20.554710+0000","last_fullsized":"2023-03-25T04:29:20.554710+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309216+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:29:28.309216+0000","last_peered":"2023-03-25T04:29:28.309216+0000","last_clean":"2023-03-25T04:29:28.309216+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:29:28.309216+0000","last_undegraded":"2023-03-25T04:29:28.309216+0000","last_fullsized":"2023-03-25T04:29:28.309216+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309233+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:29:28.309233+0000","last_peered":"2023-03-25T04:29:28.309233+0000","last_clean":"2023-03-25T04:29:28.309233+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:29:28.309233+0000","last_undegraded":"2023-03-25T04:29:28.309233+0000","last_fullsized":"2023-03-25T04:29:28.309233+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117046,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:30:37 2023","interfaces":[{"interface":"back","average":{"1min":0.28799999999999998,"5min":0.28899999999999998,"15min":0.29499999999999998},"min":{"1min":0.20200000000000001,"5min":0.191,"15min":0.191},"max":{"1min":0.433,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.216},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.28999999999999998,"15min":0.29799999999999999},"min":{"1min":0.17499999999999999,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":0.40300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.24299999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051637,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:31:35 2023","interfaces":[{"interface":"back","average":{"1min":0.26100000000000001,"5min":0.25700000000000001,"15min":0.251},"min":{"1min":0.19,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.35899999999999999,"5min":0.41599999999999998,"15min":0.41599999999999998},"last":0.221},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.27400000000000002,"15min":0.26700000000000002},"min":{"1min":0.20200000000000001,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.374,"5min":0.42899999999999999,"15min":0.42899999999999999},"last":0.24099999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:32:01.906+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:32:02.388+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:32:02.794 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:32:22.326 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":233,"stamp":"2023-03-25T04:32:22.153289+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002545"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:33.309748+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:29:33.309748+0000","last_peered":"2023-03-25T04:29:33.309748+0000","last_clean":"2023-03-25T04:29:33.309748+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:29:33.309748+0000","last_undegraded":"2023-03-25T04:29:33.309748+0000","last_fullsized":"2023-03-25T04:29:33.309748+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:33.309729+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:29:33.309729+0000","last_peered":"2023-03-25T04:29:33.309729+0000","last_clean":"2023-03-25T04:29:33.309729+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:29:33.309729+0000","last_undegraded":"2023-03-25T04:29:33.309729+0000","last_fullsized":"2023-03-25T04:29:33.309729+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:23.308607+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:29:23.308607+0000","last_peered":"2023-03-25T04:29:23.308607+0000","last_clean":"2023-03-25T04:29:23.308607+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:29:23.308607+0000","last_undegraded":"2023-03-25T04:29:23.308607+0000","last_fullsized":"2023-03-25T04:29:23.308607+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":28,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:20.554710+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:29:20.554710+0000","last_peered":"2023-03-25T04:29:20.554710+0000","last_clean":"2023-03-25T04:29:20.554710+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:29:20.554710+0000","last_undegraded":"2023-03-25T04:29:20.554710+0000","last_fullsized":"2023-03-25T04:29:20.554710+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309216+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:29:28.309216+0000","last_peered":"2023-03-25T04:29:28.309216+0000","last_clean":"2023-03-25T04:29:28.309216+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:29:28.309216+0000","last_undegraded":"2023-03-25T04:29:28.309216+0000","last_fullsized":"2023-03-25T04:29:28.309216+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309233+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:29:28.309233+0000","last_peered":"2023-03-25T04:29:28.309233+0000","last_clean":"2023-03-25T04:29:28.309233+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:29:28.309233+0000","last_undegraded":"2023-03-25T04:29:28.309233+0000","last_fullsized":"2023-03-25T04:29:28.309233+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117055,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:31:42 2023","interfaces":[{"interface":"back","average":{"1min":0.27500000000000002,"5min":0.28399999999999997,"15min":0.29399999999999998},"min":{"1min":0.19800000000000001,"5min":0.191,"15min":0.191},"max":{"1min":0.36899999999999999,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.32300000000000001},{"interface":"front","average":{"1min":0.27900000000000003,"5min":0.28499999999999998,"15min":0.29599999999999999},"min":{"1min":0.216,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":0.40799999999999997,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.27100000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051646,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:31:35 2023","interfaces":[{"interface":"back","average":{"1min":0.26100000000000001,"5min":0.25700000000000001,"15min":0.251},"min":{"1min":0.19,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.35899999999999999,"5min":0.41599999999999998,"15min":0.41599999999999998},"last":0.25600000000000001},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.27400000000000002,"15min":0.26700000000000002},"min":{"1min":0.20200000000000001,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.374,"5min":0.42899999999999999,"15min":0.42899999999999999},"last":0.29199999999999998}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:32:22.327 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean
, 2023-03-25T04:32:22.327 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}

Ignoring malparsed backtrace: 2023-03-25T04:32:39.836 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:32:39.849 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":241,"stamp":"2023-03-25T04:32:38.156312+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002346"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:33.309748+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:29:33.309748+0000","last_peered":"2023-03-25T04:29:33.309748+0000","last_clean":"2023-03-25T04:29:33.309748+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:29:33.309748+0000","last_undegraded":"2023-03-25T04:29:33.309748+0000","last_fullsized":"2023-03-25T04:29:33.309748+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:33.309729+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:29:33.309729+0000","last_peered":"2023-03-25T04:29:33.309729+0000","last_clean":"2023-03-25T04:29:33.309729+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:29:33.309729+0000","last_undegraded":"2023-03-25T04:29:33.309729+0000","last_fullsized":"2023-03-25T04:29:33.309729+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:23.308607+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:29:23.308607+0000","last_peered":"2023-03-25T04:29:23.308607+0000","last_clean":"2023-03-25T04:29:23.308607+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:29:23.308607+0000","last_undegraded":"2023-03-25T04:29:23.308607+0000","last_fullsized":"2023-03-25T04:29:23.308607+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":28,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:20.554710+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:29:20.554710+0000","last_peered":"2023-03-25T04:29:20.554710+0000","last_clean":"2023-03-25T04:29:20.554710+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:29:20.554710+0000","last_undegraded":"2023-03-25T04:29:20.554710+0000","last_fullsized":"2023-03-25T04:29:20.554710+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309216+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:29:28.309216+0000","last_peered":"2023-03-25T04:29:28.309216+0000","last_clean":"2023-03-25T04:29:28.309216+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:29:28.309216+0000","last_undegraded":"2023-03-25T04:29:28.309216+0000","last_fullsized":"2023-03-25T04:29:28.309216+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309233+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:29:28.309233+0000","last_peered":"2023-03-25T04:29:28.309233+0000","last_clean":"2023-03-25T04:29:28.309233+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:29:28.309233+0000","last_undegraded":"2023-03-25T04:29:28.309233+0000","last_fullsized":"2023-03-25T04:29:28.309233+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117058,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:31:42 2023","interfaces":[{"interface":"back","average":{"1min":0.27500000000000002,"5min":0.28399999999999997,"15min":0.29399999999999998},"min":{"1min":0.19800000000000001,"5min":0.191,"15min":0.191},"max":{"1min":0.36899999999999999,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.30599999999999999},{"interface":"front","average":{"1min":0.27900000000000003,"5min":0.28499999999999998,"15min":0.29599999999999999},"min":{"1min":0.216,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":0.40799999999999997,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.32500000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051649,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:31:35 2023","interfaces":[{"interface":"back","average":{"1min":0.26100000000000001,"5min":0.25700000000000001,"15min":0.251},"min":{"1min":0.19,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.35899999999999999,"5min":0.41599999999999998,"15min":0.41599999999999998},"last":0.24199999999999999},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.27400000000000002,"15min":0.26700000000000002},"min":{"1min":0.20200000000000001,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.374,"5min":0.42899999999999999,"15min":0.42899999999999999},"last":0.22600000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:32:54.944+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:32:55.469+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:32:55.927+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)

Ignoring malparsed backtrace: 2023-03-25T04:33:13.958 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean
, 2023-03-25T04:33:13.959 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}
, 2023-03-25T04:33:13.959 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json

Ignoring malparsed backtrace: 2023-03-25T04:33:31.199 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":267,"stamp":"2023-03-25T04:33:30.166660+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002269"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:33.309748+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:29:33.309748+0000","last_peered":"2023-03-25T04:29:33.309748+0000","last_clean":"2023-03-25T04:29:33.309748+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:29:33.309748+0000","last_undegraded":"2023-03-25T04:29:33.309748+0000","last_fullsized":"2023-03-25T04:29:33.309748+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:33.309729+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:29:33.309729+0000","last_peered":"2023-03-25T04:29:33.309729+0000","last_clean":"2023-03-25T04:29:33.309729+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:29:33.309729+0000","last_undegraded":"2023-03-25T04:29:33.309729+0000","last_fullsized":"2023-03-25T04:29:33.309729+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":28,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:31:23.320894+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:31:23.320894+0000","last_peered":"2023-03-25T04:31:23.320894+0000","last_clean":"2023-03-25T04:31:23.320894+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:31:23.320894+0000","last_undegraded":"2023-03-25T04:31:23.320894+0000","last_fullsized":"2023-03-25T04:31:23.320894+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":28,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:20.554710+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:29:20.554710+0000","last_peered":"2023-03-25T04:29:20.554710+0000","last_clean":"2023-03-25T04:29:20.554710+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:29:20.554710+0000","last_undegraded":"2023-03-25T04:29:20.554710+0000","last_fullsized":"2023-03-25T04:29:20.554710+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309216+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:29:28.309216+0000","last_peered":"2023-03-25T04:29:28.309216+0000","last_clean":"2023-03-25T04:29:28.309216+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:29:28.309216+0000","last_undegraded":"2023-03-25T04:29:28.309216+0000","last_fullsized":"2023-03-25T04:29:28.309216+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309233+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:29:28.309233+0000","last_peered":"2023-03-25T04:29:28.309233+0000","last_clean":"2023-03-25T04:29:28.309233+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:29:28.309233+0000","last_undegraded":"2023-03-25T04:29:28.309233+0000","last_fullsized":"2023-03-25T04:29:28.309233+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117068,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:32:46 2023","interfaces":[{"interface":"back","average":{"1min":0.27000000000000002,"5min":0.27900000000000003,"15min":0.29199999999999998},"min":{"1min":0.17899999999999999,"5min":0.17899999999999999,"15min":0.17899999999999999},"max":{"1min":0.35599999999999998,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.28799999999999998},{"interface":"front","average":{"1min":0.26800000000000002,"5min":0.27800000000000002,"15min":0.29399999999999998},"min":{"1min":0.19900000000000001,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":0.377,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.315}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051660,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:32:36 2023","interfaces":[{"interface":"back","average":{"1min":0.248,"5min":0.25700000000000001,"15min":0.251},"min":{"1min":0.17000000000000001,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"last":0.22700000000000001},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.27600000000000002,"15min":0.26800000000000002},"min":{"1min":0.192,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"last":0.20399999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:33:31.201 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:33:31.400+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)

Ignoring malparsed backtrace: 2023-03-25T04:33:47.531+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:33:47.951+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:33:48.505+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)

Ignoring malparsed backtrace: 2023-03-25T04:34:06.104 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json
, 2023-03-25T04:34:06.118+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:34:06.497 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:34:23.399 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:34:23.665+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:34:23.712 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:34:40.195+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:34:40.523+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:34:40.676 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace: 2023-03-25T04:35:03.381+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:35:04.139+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:35:04.391+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)

Ignoring malparsed backtrace: 2023-03-25T04:35:22.738 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean
, 2023-03-25T04:35:22.739 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}
, 2023-03-25T04:35:22.739 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json

Ignoring malparsed backtrace: 2023-03-25T04:35:41.393+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:35:42.352+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:35:42.376+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:35:57.384 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":340,"stamp":"2023-03-25T04:35:56.194992+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.001930"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":36,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:31:43.322894+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:31:43.322894+0000","last_peered":"2023-03-25T04:31:43.322894+0000","last_clean":"2023-03-25T04:31:43.322894+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:31:43.322894+0000","last_undegraded":"2023-03-25T04:31:43.322894+0000","last_fullsized":"2023-03-25T04:31:43.322894+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":28,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:31:43.322876+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:31:43.322876+0000","last_peered":"2023-03-25T04:31:43.322876+0000","last_clean":"2023-03-25T04:31:43.322876+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:31:43.322876+0000","last_undegraded":"2023-03-25T04:31:43.322876+0000","last_fullsized":"2023-03-25T04:31:43.322876+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:23.331998+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:33:23.331998+0000","last_peered":"2023-03-25T04:33:23.331998+0000","last_clean":"2023-03-25T04:33:23.331998+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:33:23.331998+0000","last_undegraded":"2023-03-25T04:33:23.331998+0000","last_fullsized":"2023-03-25T04:33:23.331998+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:30.576723+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:33:30.576723+0000","last_peered":"2023-03-25T04:33:30.576723+0000","last_clean":"2023-03-25T04:33:30.576723+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:33:30.576723+0000","last_undegraded":"2023-03-25T04:33:30.576723+0000","last_fullsized":"2023-03-25T04:33:30.576723+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":37,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:38.333447+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:33:38.333447+0000","last_peered":"2023-03-25T04:33:38.333447+0000","last_clean":"2023-03-25T04:33:38.333447+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:33:38.333447+0000","last_undegraded":"2023-03-25T04:33:38.333447+0000","last_fullsized":"2023-03-25T04:33:38.333447+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:38.333467+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:33:38.333467+0000","last_peered":"2023-03-25T04:33:38.333467+0000","last_clean":"2023-03-25T04:33:38.333467+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:33:38.333467+0000","last_undegraded":"2023-03-25T04:33:38.333467+0000","last_fullsized":"2023-03-25T04:33:38.333467+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117098,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:35:49 2023","interfaces":[{"interface":"back","average":{"1min":0.255,"5min":0.26000000000000001,"15min":0.28199999999999997},"min":{"1min":0.17899999999999999,"5min":0.16,"15min":0.16},"max":{"1min":0.36399999999999999,"5min":0.42599999999999999,"15min":0.45100000000000001},"last":0.26100000000000001},{"interface":"front","average":{"1min":0.26000000000000001,"5min":0.26700000000000002,"15min":0.28599999999999998},"min":{"1min":0.188,"5min":0.17999999999999999,"15min":0.17499999999999999},"max":{"1min":0.39100000000000001,"5min":0.45200000000000001,"15min":0.46300000000000002},"last":0.28399999999999997}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051689,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:35:42 2023","interfaces":[{"interface":"back","average":{"1min":0.29399999999999998,"5min":0.27200000000000002,"15min":0.25800000000000001},"min":{"1min":0.17299999999999999,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.41099999999999998,"5min":0.42699999999999999,"15min":0.42699999999999999},"last":0.20399999999999999},{"interface":"front","average":{"1min":0.30299999999999999,"5min":0.28399999999999997,"15min":0.27300000000000002},"min":{"1min":0.19700000000000001,"5min":0.192,"15min":0.16600000000000001},"max":{"1min":0.43099999999999999,"5min":0.45800000000000002,"15min":0.45800000000000002},"last":0.22}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:35:57.624+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)

Ignoring malparsed backtrace: 2023-03-25T04:36:17.226 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:36:17.239 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":350,"stamp":"2023-03-25T04:36:16.198866+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002148"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":37,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:53.335026+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:33:53.335026+0000","last_peered":"2023-03-25T04:33:53.335026+0000","last_clean":"2023-03-25T04:33:53.335026+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:33:53.335026+0000","last_undegraded":"2023-03-25T04:33:53.335026+0000","last_fullsized":"2023-03-25T04:33:53.335026+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:53.335009+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:33:53.335009+0000","last_peered":"2023-03-25T04:33:53.335009+0000","last_clean":"2023-03-25T04:33:53.335009+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:33:53.335009+0000","last_undegraded":"2023-03-25T04:33:53.335009+0000","last_fullsized":"2023-03-25T04:33:53.335009+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:23.331998+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:33:23.331998+0000","last_peered":"2023-03-25T04:33:23.331998+0000","last_clean":"2023-03-25T04:33:23.331998+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:33:23.331998+0000","last_undegraded":"2023-03-25T04:33:23.331998+0000","last_fullsized":"2023-03-25T04:33:23.331998+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:30.576723+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:33:30.576723+0000","last_peered":"2023-03-25T04:33:30.576723+0000","last_clean":"2023-03-25T04:33:30.576723+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:33:30.576723+0000","last_undegraded":"2023-03-25T04:33:30.576723+0000","last_fullsized":"2023-03-25T04:33:30.576723+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":37,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:38.333447+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:33:38.333447+0000","last_peered":"2023-03-25T04:33:38.333447+0000","last_clean":"2023-03-25T04:33:38.333447+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:33:38.333447+0000","last_undegraded":"2023-03-25T04:33:38.333447+0000","last_fullsized":"2023-03-25T04:33:38.333447+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:38.333467+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:33:38.333467+0000","last_peered":"2023-03-25T04:33:38.333467+0000","last_clean":"2023-03-25T04:33:38.333467+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:33:38.333467+0000","last_undegraded":"2023-03-25T04:33:38.333467+0000","last_fullsized":"2023-03-25T04:33:38.333467+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117102,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:35:49 2023","interfaces":[{"interface":"back","average":{"1min":0.255,"5min":0.26000000000000001,"15min":0.28199999999999997},"min":{"1min":0.17899999999999999,"5min":0.16,"15min":0.16},"max":{"1min":0.36399999999999999,"5min":0.42599999999999999,"15min":0.45100000000000001},"last":0.28399999999999997},{"interface":"front","average":{"1min":0.26000000000000001,"5min":0.26700000000000002,"15min":0.28599999999999998},"min":{"1min":0.188,"5min":0.17999999999999999,"15min":0.17499999999999999},"max":{"1min":0.39100000000000001,"5min":0.45200000000000001,"15min":0.46300000000000002},"last":0.30299999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051693,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:35:42 2023","interfaces":[{"interface":"back","average":{"1min":0.29399999999999998,"5min":0.27200000000000002,"15min":0.25800000000000001},"min":{"1min":0.17299999999999999,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.41099999999999998,"5min":0.42699999999999999,"15min":0.42699999999999999},"last":0.35099999999999998},{"interface":"front","average":{"1min":0.30299999999999999,"5min":0.28399999999999997,"15min":0.27300000000000002},"min":{"1min":0.19700000000000001,"5min":0.192,"15min":0.16600000000000001},"max":{"1min":0.43099999999999999,"5min":0.45800000000000002,"15min":0.45800000000000002},"last":0.32900000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:36:34.556+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:36:34.717+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:36:35.583+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)

Ignoring malparsed backtrace: 2023-03-25T04:36:52.680 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json
, 2023-03-25T04:36:52.701+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:36:53.067 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:37:09.963 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:37:10.273 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all

Ignoring malparsed backtrace: 2023-03-25T04:37:26.856+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:37:27.288 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:37:27.538+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:37:41.903 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":392,"stamp":"2023-03-25T04:37:40.213373+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002012"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":37,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:53.335026+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:33:53.335026+0000","last_peered":"2023-03-25T04:33:53.335026+0000","last_clean":"2023-03-25T04:33:53.335026+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:33:53.335026+0000","last_undegraded":"2023-03-25T04:33:53.335026+0000","last_fullsized":"2023-03-25T04:33:53.335026+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:53.335009+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:33:53.335009+0000","last_peered":"2023-03-25T04:33:53.335009+0000","last_clean":"2023-03-25T04:33:53.335009+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:33:53.335009+0000","last_undegraded":"2023-03-25T04:33:53.335009+0000","last_fullsized":"2023-03-25T04:33:53.335009+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:23.343585+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:35:23.343585+0000","last_peered":"2023-03-25T04:35:23.343585+0000","last_clean":"2023-03-25T04:35:23.343585+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:35:23.343585+0000","last_undegraded":"2023-03-25T04:35:23.343585+0000","last_fullsized":"2023-03-25T04:35:23.343585+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:30.576723+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:33:30.576723+0000","last_peered":"2023-03-25T04:33:30.576723+0000","last_clean":"2023-03-25T04:33:30.576723+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:33:30.576723+0000","last_undegraded":"2023-03-25T04:33:30.576723+0000","last_fullsized":"2023-03-25T04:33:30.576723+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":37,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:38.333447+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:33:38.333447+0000","last_peered":"2023-03-25T04:33:38.333447+0000","last_clean":"2023-03-25T04:33:38.333447+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:33:38.333447+0000","last_undegraded":"2023-03-25T04:33:38.333447+0000","last_fullsized":"2023-03-25T04:33:38.333447+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:38.333467+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:33:38.333467+0000","last_peered":"2023-03-25T04:33:38.333467+0000","last_clean":"2023-03-25T04:33:38.333467+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:33:38.333467+0000","last_undegraded":"2023-03-25T04:33:38.333467+0000","last_fullsized":"2023-03-25T04:33:38.333467+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117119,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:36:50 2023","interfaces":[{"interface":"back","average":{"1min":0.254,"5min":0.25600000000000001,"15min":0.27900000000000003},"min":{"1min":0.20499999999999999,"5min":0.16,"15min":0.16},"max":{"1min":0.371,"5min":0.42599999999999999,"15min":0.45100000000000001},"last":0.23000000000000001},{"interface":"front","average":{"1min":0.26800000000000002,"5min":0.26500000000000001,"15min":0.28399999999999997},"min":{"1min":0.19700000000000001,"5min":0.17999999999999999,"15min":0.17499999999999999},"max":{"1min":0.34699999999999998,"5min":0.45200000000000001,"15min":0.46300000000000002},"last":0.20899999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051710,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:36:44 2023","interfaces":[{"interface":"back","average":{"1min":0.28399999999999997,"5min":0.27700000000000002,"15min":0.26100000000000001},"min":{"1min":0.192,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.40899999999999997,"5min":0.42699999999999999,"15min":0.42699999999999999},"last":0.39700000000000002},{"interface":"front","average":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.27500000000000002},"min":{"1min":0.182,"5min":0.182,"15min":0.16600000000000001},"max":{"1min":0.439,"5min":0.45800000000000002,"15min":0.45800000000000002},"last":0.439}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:37:42.400+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)

Ignoring malparsed backtrace: 2023-03-25T04:38:04.766+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:38:04.922 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:38:05.246 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:38:19.605 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":411,"stamp":"2023-03-25T04:38:18.220966+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002368"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":38,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:36:03.347070+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:36:03.347070+0000","last_peered":"2023-03-25T04:36:03.347070+0000","last_clean":"2023-03-25T04:36:03.347070+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:36:03.347070+0000","last_undegraded":"2023-03-25T04:36:03.347070+0000","last_fullsized":"2023-03-25T04:36:03.347070+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:36:03.347025+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:36:03.347025+0000","last_peered":"2023-03-25T04:36:03.347025+0000","last_clean":"2023-03-25T04:36:03.347025+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:36:03.347025+0000","last_undegraded":"2023-03-25T04:36:03.347025+0000","last_fullsized":"2023-03-25T04:36:03.347025+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:23.343585+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:35:23.343585+0000","last_peered":"2023-03-25T04:35:23.343585+0000","last_clean":"2023-03-25T04:35:23.343585+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:35:23.343585+0000","last_undegraded":"2023-03-25T04:35:23.343585+0000","last_fullsized":"2023-03-25T04:35:23.343585+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:35.587025+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:35:35.587025+0000","last_peered":"2023-03-25T04:35:35.587025+0000","last_clean":"2023-03-25T04:35:35.587025+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:35:35.587025+0000","last_undegraded":"2023-03-25T04:35:35.587025+0000","last_fullsized":"2023-03-25T04:35:35.587025+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":38,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:43.345308+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:35:43.345308+0000","last_peered":"2023-03-25T04:35:43.345308+0000","last_clean":"2023-03-25T04:35:43.345308+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:35:43.345308+0000","last_undegraded":"2023-03-25T04:35:43.345308+0000","last_fullsized":"2023-03-25T04:35:43.345308+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:43.345324+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:35:43.345324+0000","last_peered":"2023-03-25T04:35:43.345324+0000","last_clean":"2023-03-25T04:35:43.345324+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:35:43.345324+0000","last_undegraded":"2023-03-25T04:35:43.345324+0000","last_fullsized":"2023-03-25T04:35:43.345324+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117127,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:37:53 2023","interfaces":[{"interface":"back","average":{"1min":0.25800000000000001,"5min":0.253,"15min":0.27700000000000002},"min":{"1min":0.184,"5min":0.16,"15min":0.16},"max":{"1min":0.44600000000000001,"5min":0.44600000000000001,"15min":0.45100000000000001},"last":0.35199999999999998},{"interface":"front","average":{"1min":0.26900000000000002,"5min":0.26500000000000001,"15min":0.28199999999999997},"min":{"1min":0.192,"5min":0.17999999999999999,"15min":0.17499999999999999},"max":{"1min":0.46400000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"last":0.33200000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051717,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:37:44 2023","interfaces":[{"interface":"back","average":{"1min":0.27300000000000002,"5min":0.28199999999999997,"15min":0.26200000000000001},"min":{"1min":0.17899999999999999,"5min":0.17299999999999999,"15min":0.17000000000000001},"max":{"1min":0.41799999999999998,"5min":0.42099999999999999,"15min":0.42699999999999999},"last":0.28699999999999998},{"interface":"front","average":{"1min":0.29399999999999998,"5min":0.29099999999999998,"15min":0.27700000000000002},"min":{"1min":0.19900000000000001,"5min":0.182,"15min":0.16600000000000001},"max":{"1min":0.441,"5min":0.441,"15min":0.45800000000000002},"last":0.26300000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:38:19.718+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:38:20.315+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)

Ignoring malparsed backtrace: 2023-03-25T04:38:44.283 INFO:teuthology.orchestra.run.smithi070.stdout:
, 2023-03-25T04:38:44.283 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":781,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":16,"modified":"2023-03-25T04:38:40.225321+0000","services":{}},"progress_events":{}}
, 2023-03-25T04:38:44.296 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now

Ignoring malparsed backtrace: 2023-03-25T04:39:02.477+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:39:02.578+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:39:03.524+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:39:20.359 INFO:teuthology.orchestra.run.smithi070.stdout:
, 2023-03-25T04:39:20.359 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":818,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":16,"modified":"2023-03-25T04:38:40.225321+0000","services":{}},"progress_events":{}}
, 2023-03-25T04:39:20.372 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now

Ignoring malparsed backtrace: 2023-03-25T04:39:37.527 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:39:37.541 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":450,"stamp":"2023-03-25T04:39:36.235332+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002086"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":38,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:36:03.347070+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:36:03.347070+0000","last_peered":"2023-03-25T04:36:03.347070+0000","last_clean":"2023-03-25T04:36:03.347070+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:36:03.347070+0000","last_undegraded":"2023-03-25T04:36:03.347070+0000","last_fullsized":"2023-03-25T04:36:03.347070+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:36:03.347025+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:36:03.347025+0000","last_peered":"2023-03-25T04:36:03.347025+0000","last_clean":"2023-03-25T04:36:03.347025+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:36:03.347025+0000","last_undegraded":"2023-03-25T04:36:03.347025+0000","last_fullsized":"2023-03-25T04:36:03.347025+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:23.353778+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:37:23.353778+0000","last_peered":"2023-03-25T04:37:23.353778+0000","last_clean":"2023-03-25T04:37:23.353778+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:37:23.353778+0000","last_undegraded":"2023-03-25T04:37:23.353778+0000","last_fullsized":"2023-03-25T04:37:23.353778+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:35.587025+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:35:35.587025+0000","last_peered":"2023-03-25T04:35:35.587025+0000","last_clean":"2023-03-25T04:35:35.587025+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:35:35.587025+0000","last_undegraded":"2023-03-25T04:35:35.587025+0000","last_fullsized":"2023-03-25T04:35:35.587025+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":38,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:43.345308+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:35:43.345308+0000","last_peered":"2023-03-25T04:35:43.345308+0000","last_clean":"2023-03-25T04:35:43.345308+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:35:43.345308+0000","last_undegraded":"2023-03-25T04:35:43.345308+0000","last_fullsized":"2023-03-25T04:35:43.345308+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:43.345324+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:35:43.345324+0000","last_peered":"2023-03-25T04:35:43.345324+0000","last_clean":"2023-03-25T04:35:43.345324+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:35:43.345324+0000","last_undegraded":"2023-03-25T04:35:43.345324+0000","last_fullsized":"2023-03-25T04:35:43.345324+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117143,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:38:54 2023","interfaces":[{"interface":"back","average":{"1min":0.25800000000000001,"5min":0.25700000000000001,"15min":0.27400000000000002},"min":{"1min":0.192,"5min":0.17399999999999999,"15min":0.16},"max":{"1min":0.39400000000000002,"5min":0.44600000000000001,"15min":0.45100000000000001},"last":0.26500000000000001},{"interface":"front","average":{"1min":0.25900000000000001,"5min":0.26700000000000002,"15min":0.27900000000000003},"min":{"1min":0.19500000000000001,"5min":0.188,"15min":0.17499999999999999},"max":{"1min":0.41499999999999998,"5min":0.46400000000000002,"15min":0.46400000000000002},"last":0.29099999999999998}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051733,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:38:45 2023","interfaces":[{"interface":"back","average":{"1min":0.27300000000000002,"5min":0.28100000000000003,"15min":0.26400000000000001},"min":{"1min":0.19400000000000001,"5min":0.17299999999999999,"15min":0.17000000000000001},"max":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"last":0.27300000000000002},{"interface":"front","average":{"1min":0.28799999999999998,"5min":0.29099999999999998,"15min":0.27800000000000002},"min":{"1min":0.19800000000000001,"5min":0.182,"15min":0.16600000000000001},"max":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"last":0.308}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:39:54.416+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:39:54.455 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:39:54.780 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:40:09.288+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:40:09.389+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:40:10.267+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)

Ignoring malparsed backtrace: 2023-03-25T04:40:28.984 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":476,"stamp":"2023-03-25T04:40:28.245911+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002693"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":39,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:38:13.358968+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:38:13.358968+0000","last_peered":"2023-03-25T04:38:13.358968+0000","last_clean":"2023-03-25T04:38:13.358968+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:38:13.358968+0000","last_undegraded":"2023-03-25T04:38:13.358968+0000","last_fullsized":"2023-03-25T04:38:13.358968+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:38:13.358922+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:38:13.358922+0000","last_peered":"2023-03-25T04:38:13.358922+0000","last_clean":"2023-03-25T04:38:13.358922+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:38:13.358922+0000","last_undegraded":"2023-03-25T04:38:13.358922+0000","last_fullsized":"2023-03-25T04:38:13.358922+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:23.353778+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:37:23.353778+0000","last_peered":"2023-03-25T04:37:23.353778+0000","last_clean":"2023-03-25T04:37:23.353778+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:37:23.353778+0000","last_undegraded":"2023-03-25T04:37:23.353778+0000","last_fullsized":"2023-03-25T04:37:23.353778+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:40.598579+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:37:40.598579+0000","last_peered":"2023-03-25T04:37:40.598579+0000","last_clean":"2023-03-25T04:37:40.598579+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:37:40.598579+0000","last_undegraded":"2023-03-25T04:37:40.598579+0000","last_fullsized":"2023-03-25T04:37:40.598579+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":39,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:48.356344+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:37:48.356344+0000","last_peered":"2023-03-25T04:37:48.356344+0000","last_clean":"2023-03-25T04:37:48.356344+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:37:48.356344+0000","last_undegraded":"2023-03-25T04:37:48.356344+0000","last_fullsized":"2023-03-25T04:37:48.356344+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:48.356365+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:37:48.356365+0000","last_peered":"2023-03-25T04:37:48.356365+0000","last_clean":"2023-03-25T04:37:48.356365+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:37:48.356365+0000","last_undegraded":"2023-03-25T04:37:48.356365+0000","last_fullsized":"2023-03-25T04:37:48.356365+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117153,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:39:54 2023","interfaces":[{"interface":"back","average":{"1min":0.25900000000000001,"5min":0.25600000000000001,"15min":0.27100000000000002},"min":{"1min":0.188,"5min":0.17899999999999999,"15min":0.16},"max":{"1min":0.41999999999999998,"5min":0.44600000000000001,"15min":0.45100000000000001},"last":0.255},{"interface":"front","average":{"1min":0.27500000000000002,"5min":0.26600000000000001,"15min":0.27700000000000002},"min":{"1min":0.17699999999999999,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.45600000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"last":0.26800000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051743,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:39:47 2023","interfaces":[{"interface":"back","average":{"1min":0.26300000000000001,"5min":0.27700000000000002,"15min":0.26500000000000001},"min":{"1min":0.187,"5min":0.17299999999999999,"15min":0.17000000000000001},"max":{"1min":0.433,"5min":0.433,"15min":0.433},"last":0.36599999999999999},{"interface":"front","average":{"1min":0.27100000000000002,"5min":0.28799999999999998,"15min":0.27900000000000003},"min":{"1min":0.20100000000000001,"5min":0.182,"15min":0.16600000000000001},"max":{"1min":0.48599999999999999,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.34999999999999998}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:40:28.986 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean
, 2023-03-25T04:40:28.986 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}

Ignoring malparsed backtrace: 2023-03-25T04:40:47.229+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:40:47.503+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:40:48.251+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)

Ignoring malparsed backtrace: 2023-03-25T04:41:03.267 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:41:03.578 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all

Ignoring malparsed backtrace: 2023-03-25T04:41:21.347 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":502,"stamp":"2023-03-25T04:41:20.256204+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002342"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":39,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:38:13.358968+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:38:13.358968+0000","last_peered":"2023-03-25T04:38:13.358968+0000","last_clean":"2023-03-25T04:38:13.358968+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:38:13.358968+0000","last_undegraded":"2023-03-25T04:38:13.358968+0000","last_fullsized":"2023-03-25T04:38:13.358968+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:38:13.358922+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:38:13.358922+0000","last_peered":"2023-03-25T04:38:13.358922+0000","last_clean":"2023-03-25T04:38:13.358922+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:38:13.358922+0000","last_undegraded":"2023-03-25T04:38:13.358922+0000","last_fullsized":"2023-03-25T04:38:13.358922+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:23.353778+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:37:23.353778+0000","last_peered":"2023-03-25T04:37:23.353778+0000","last_clean":"2023-03-25T04:37:23.353778+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:37:23.353778+0000","last_undegraded":"2023-03-25T04:37:23.353778+0000","last_fullsized":"2023-03-25T04:37:23.353778+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:40.598579+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:37:40.598579+0000","last_peered":"2023-03-25T04:37:40.598579+0000","last_clean":"2023-03-25T04:37:40.598579+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:37:40.598579+0000","last_undegraded":"2023-03-25T04:37:40.598579+0000","last_fullsized":"2023-03-25T04:37:40.598579+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":39,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:48.356344+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:37:48.356344+0000","last_peered":"2023-03-25T04:37:48.356344+0000","last_clean":"2023-03-25T04:37:48.356344+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:37:48.356344+0000","last_undegraded":"2023-03-25T04:37:48.356344+0000","last_fullsized":"2023-03-25T04:37:48.356344+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:48.356365+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:37:48.356365+0000","last_peered":"2023-03-25T04:37:48.356365+0000","last_clean":"2023-03-25T04:37:48.356365+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:37:48.356365+0000","last_undegraded":"2023-03-25T04:37:48.356365+0000","last_fullsized":"2023-03-25T04:37:48.356365+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117163,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:40:56 2023","interfaces":[{"interface":"back","average":{"1min":0.28799999999999998,"5min":0.26300000000000001,"15min":0.27100000000000002},"min":{"1min":0.188,"5min":0.184,"15min":0.16},"max":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.17499999999999999},{"interface":"front","average":{"1min":0.29599999999999999,"5min":0.27300000000000002,"15min":0.27700000000000002},"min":{"1min":0.20599999999999999,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.43099999999999999,"5min":0.46400000000000002,"15min":0.46400000000000002},"last":0.20699999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051754,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:40:52 2023","interfaces":[{"interface":"back","average":{"1min":0.28599999999999998,"5min":0.27500000000000002,"15min":0.26700000000000002},"min":{"1min":0.17899999999999999,"5min":0.17899999999999999,"15min":0.17000000000000001},"max":{"1min":0.44,"5min":0.44,"15min":0.44},"last":0.23699999999999999},{"interface":"front","average":{"1min":0.29799999999999999,"5min":0.28699999999999998,"15min":0.28100000000000003},"min":{"1min":0.20300000000000001,"5min":0.182,"15min":0.16600000000000001},"max":{"1min":0.47599999999999998,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.28000000000000003}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:41:21.349 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean
, 2023-03-25T04:41:21.349 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}

Ignoring malparsed backtrace: 2023-03-25T04:41:39.867+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:41:40.383+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:41:40.905+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:41:58.219 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json
, 2023-03-25T04:41:58.319+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:41:58.644 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:42:18.193 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":530,"stamp":"2023-03-25T04:42:16.266659+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002356"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":39,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:38:13.358968+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:38:13.358968+0000","last_peered":"2023-03-25T04:38:13.358968+0000","last_clean":"2023-03-25T04:38:13.358968+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:38:13.358968+0000","last_undegraded":"2023-03-25T04:38:13.358968+0000","last_fullsized":"2023-03-25T04:38:13.358968+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:38:13.358922+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:38:13.358922+0000","last_peered":"2023-03-25T04:38:13.358922+0000","last_clean":"2023-03-25T04:38:13.358922+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:38:13.358922+0000","last_undegraded":"2023-03-25T04:38:13.358922+0000","last_fullsized":"2023-03-25T04:38:13.358922+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:23.364593+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:39:23.364593+0000","last_peered":"2023-03-25T04:39:23.364593+0000","last_clean":"2023-03-25T04:39:23.364593+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:39:23.364593+0000","last_undegraded":"2023-03-25T04:39:23.364593+0000","last_fullsized":"2023-03-25T04:39:23.364593+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:45.608475+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:39:45.608475+0000","last_peered":"2023-03-25T04:39:45.608475+0000","last_clean":"2023-03-25T04:39:45.608475+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:39:45.608475+0000","last_undegraded":"2023-03-25T04:39:45.608475+0000","last_fullsized":"2023-03-25T04:39:45.608475+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":40,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:53.367398+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:39:53.367398+0000","last_peered":"2023-03-25T04:39:53.367398+0000","last_clean":"2023-03-25T04:39:53.367398+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:39:53.367398+0000","last_undegraded":"2023-03-25T04:39:53.367398+0000","last_fullsized":"2023-03-25T04:39:53.367398+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:53.367424+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:39:53.367424+0000","last_peered":"2023-03-25T04:39:53.367424+0000","last_clean":"2023-03-25T04:39:53.367424+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:39:53.367424+0000","last_undegraded":"2023-03-25T04:39:53.367424+0000","last_fullsized":"2023-03-25T04:39:53.367424+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117175,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:41:58 2023","interfaces":[{"interface":"back","average":{"1min":0.25800000000000001,"5min":0.26400000000000001,"15min":0.26800000000000002},"min":{"1min":0.17499999999999999,"5min":0.17499999999999999,"15min":0.16},"max":{"1min":0.39100000000000001,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.22900000000000001},{"interface":"front","average":{"1min":0.26300000000000001,"5min":0.27200000000000002,"15min":0.27400000000000002},"min":{"1min":0.20599999999999999,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.35699999999999998,"5min":0.46400000000000002,"15min":0.46400000000000002},"last":0.26400000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051765,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:41:52 2023","interfaces":[{"interface":"back","average":{"1min":0.26500000000000001,"5min":0.27200000000000002,"15min":0.26800000000000002},"min":{"1min":0.17000000000000001,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.39200000000000002,"5min":0.44,"15min":0.44},"last":0.23899999999999999},{"interface":"front","average":{"1min":0.28100000000000003,"5min":0.28599999999999998,"15min":0.28199999999999997},"min":{"1min":0.19400000000000001,"5min":0.19400000000000001,"15min":0.16600000000000001},"max":{"1min":0.41399999999999998,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.254}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:42:18.297+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:42:18.958+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:42:35.512 INFO:teuthology.orchestra.run.smithi070.stdout:
, 2023-03-25T04:42:35.512 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":1013,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":17,"modified":"2023-03-25T04:41:54.262744+0000","services":{}},"progress_events":{}}
, 2023-03-25T04:42:35.525 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:42:52.741 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":548,"stamp":"2023-03-25T04:42:52.273601+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002221"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":40,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:40:23.370056+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:40:23.370056+0000","last_peered":"2023-03-25T04:40:23.370056+0000","last_clean":"2023-03-25T04:40:23.370056+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:40:23.370056+0000","last_undegraded":"2023-03-25T04:40:23.370056+0000","last_fullsized":"2023-03-25T04:40:23.370056+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:40:23.370039+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:40:23.370039+0000","last_peered":"2023-03-25T04:40:23.370039+0000","last_clean":"2023-03-25T04:40:23.370039+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:40:23.370039+0000","last_undegraded":"2023-03-25T04:40:23.370039+0000","last_fullsized":"2023-03-25T04:40:23.370039+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:23.364593+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:39:23.364593+0000","last_peered":"2023-03-25T04:39:23.364593+0000","last_clean":"2023-03-25T04:39:23.364593+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:39:23.364593+0000","last_undegraded":"2023-03-25T04:39:23.364593+0000","last_fullsized":"2023-03-25T04:39:23.364593+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:45.608475+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:39:45.608475+0000","last_peered":"2023-03-25T04:39:45.608475+0000","last_clean":"2023-03-25T04:39:45.608475+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:39:45.608475+0000","last_undegraded":"2023-03-25T04:39:45.608475+0000","last_fullsized":"2023-03-25T04:39:45.608475+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":40,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:53.367398+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:39:53.367398+0000","last_peered":"2023-03-25T04:39:53.367398+0000","last_clean":"2023-03-25T04:39:53.367398+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:39:53.367398+0000","last_undegraded":"2023-03-25T04:39:53.367398+0000","last_fullsized":"2023-03-25T04:39:53.367398+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:53.367424+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:39:53.367424+0000","last_peered":"2023-03-25T04:39:53.367424+0000","last_clean":"2023-03-25T04:39:53.367424+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:39:53.367424+0000","last_undegraded":"2023-03-25T04:39:53.367424+0000","last_fullsized":"2023-03-25T04:39:53.367424+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117182,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:41:58 2023","interfaces":[{"interface":"back","average":{"1min":0.25800000000000001,"5min":0.26400000000000001,"15min":0.26800000000000002},"min":{"1min":0.17499999999999999,"5min":0.17499999999999999,"15min":0.16},"max":{"1min":0.39100000000000001,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.19600000000000001},{"interface":"front","average":{"1min":0.26300000000000001,"5min":0.27200000000000002,"15min":0.27400000000000002},"min":{"1min":0.20599999999999999,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.35699999999999998,"5min":0.46400000000000002,"15min":0.46400000000000002},"last":0.219}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051773,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:41:52 2023","interfaces":[{"interface":"back","average":{"1min":0.26500000000000001,"5min":0.27200000000000002,"15min":0.26800000000000002},"min":{"1min":0.17000000000000001,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.39200000000000002,"5min":0.44,"15min":0.44},"last":0.19},{"interface":"front","average":{"1min":0.28100000000000003,"5min":0.28599999999999998,"15min":0.28199999999999997},"min":{"1min":0.19400000000000001,"5min":0.19400000000000001,"15min":0.16600000000000001},"max":{"1min":0.41399999999999998,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.17599999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:42:52.742 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean

Ignoring malparsed backtrace: 2023-03-25T04:43:09.973 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:43:09.986 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":556,"stamp":"2023-03-25T04:43:08.276556+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002359"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":40,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:40:23.370056+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:40:23.370056+0000","last_peered":"2023-03-25T04:40:23.370056+0000","last_clean":"2023-03-25T04:40:23.370056+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:40:23.370056+0000","last_undegraded":"2023-03-25T04:40:23.370056+0000","last_fullsized":"2023-03-25T04:40:23.370056+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:40:23.370039+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:40:23.370039+0000","last_peered":"2023-03-25T04:40:23.370039+0000","last_clean":"2023-03-25T04:40:23.370039+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:40:23.370039+0000","last_undegraded":"2023-03-25T04:40:23.370039+0000","last_fullsized":"2023-03-25T04:40:23.370039+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:23.364593+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:39:23.364593+0000","last_peered":"2023-03-25T04:39:23.364593+0000","last_clean":"2023-03-25T04:39:23.364593+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:39:23.364593+0000","last_undegraded":"2023-03-25T04:39:23.364593+0000","last_fullsized":"2023-03-25T04:39:23.364593+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:45.608475+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:39:45.608475+0000","last_peered":"2023-03-25T04:39:45.608475+0000","last_clean":"2023-03-25T04:39:45.608475+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:39:45.608475+0000","last_undegraded":"2023-03-25T04:39:45.608475+0000","last_fullsized":"2023-03-25T04:39:45.608475+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":40,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:53.367398+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:39:53.367398+0000","last_peered":"2023-03-25T04:39:53.367398+0000","last_clean":"2023-03-25T04:39:53.367398+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:39:53.367398+0000","last_undegraded":"2023-03-25T04:39:53.367398+0000","last_fullsized":"2023-03-25T04:39:53.367398+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:53.367424+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:39:53.367424+0000","last_peered":"2023-03-25T04:39:53.367424+0000","last_clean":"2023-03-25T04:39:53.367424+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:39:53.367424+0000","last_undegraded":"2023-03-25T04:39:53.367424+0000","last_fullsized":"2023-03-25T04:39:53.367424+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117185,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:43:00 2023","interfaces":[{"interface":"back","average":{"1min":0.27400000000000002,"5min":0.26700000000000002,"15min":0.26600000000000001},"min":{"1min":0.19600000000000001,"5min":0.17499999999999999,"15min":0.16},"max":{"1min":0.42999999999999999,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.27800000000000002},{"interface":"front","average":{"1min":0.28199999999999997,"5min":0.27500000000000002,"15min":0.27300000000000002},"min":{"1min":0.185,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.47199999999999998,"5min":0.47199999999999998,"15min":0.47199999999999998},"last":0.247}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051776,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:42:54 2023","interfaces":[{"interface":"back","average":{"1min":0.26300000000000001,"5min":0.27000000000000002,"15min":0.26900000000000002},"min":{"1min":0.17399999999999999,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.438,"5min":0.44,"15min":0.44},"last":0.20100000000000001},{"interface":"front","average":{"1min":0.27400000000000002,"5min":0.28199999999999997,"15min":0.28299999999999997},"min":{"1min":0.17599999999999999,"5min":0.17599999999999999,"15min":0.16600000000000001},"max":{"1min":0.41599999999999998,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.23499999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:43:25.042+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:43:25.411+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:43:26.013+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:43:47.937+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:43:48.534+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:43:48.889+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:44:09.318 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:44:09.331 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":586,"stamp":"2023-03-25T04:44:08.288933+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002697"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":40,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:40:23.370056+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:40:23.370056+0000","last_peered":"2023-03-25T04:40:23.370056+0000","last_clean":"2023-03-25T04:40:23.370056+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:40:23.370056+0000","last_undegraded":"2023-03-25T04:40:23.370056+0000","last_fullsized":"2023-03-25T04:40:23.370056+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:40:23.370039+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:40:23.370039+0000","last_peered":"2023-03-25T04:40:23.370039+0000","last_clean":"2023-03-25T04:40:23.370039+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:40:23.370039+0000","last_undegraded":"2023-03-25T04:40:23.370039+0000","last_fullsized":"2023-03-25T04:40:23.370039+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:23.375106+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:41:23.375106+0000","last_peered":"2023-03-25T04:41:23.375106+0000","last_clean":"2023-03-25T04:41:23.375106+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:41:23.375106+0000","last_undegraded":"2023-03-25T04:41:23.375106+0000","last_fullsized":"2023-03-25T04:41:23.375106+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:50.618930+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:41:50.618930+0000","last_peered":"2023-03-25T04:41:50.618930+0000","last_clean":"2023-03-25T04:41:50.618930+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:41:50.618930+0000","last_undegraded":"2023-03-25T04:41:50.618930+0000","last_fullsized":"2023-03-25T04:41:50.618930+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":41,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:58.377821+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:41:58.377821+0000","last_peered":"2023-03-25T04:41:58.377821+0000","last_clean":"2023-03-25T04:41:58.377821+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:41:58.377821+0000","last_undegraded":"2023-03-25T04:41:58.377821+0000","last_fullsized":"2023-03-25T04:41:58.377821+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:58.377836+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:41:58.377836+0000","last_peered":"2023-03-25T04:41:58.377836+0000","last_clean":"2023-03-25T04:41:58.377836+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:41:58.377836+0000","last_undegraded":"2023-03-25T04:41:58.377836+0000","last_fullsized":"2023-03-25T04:41:58.377836+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117197,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:44:01 2023","interfaces":[{"interface":"back","average":{"1min":0.27200000000000002,"5min":0.27000000000000002,"15min":0.26400000000000001},"min":{"1min":0.20399999999999999,"5min":0.17499999999999999,"15min":0.16},"max":{"1min":0.38400000000000001,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.245},{"interface":"front","average":{"1min":0.27600000000000002,"5min":0.27800000000000002,"15min":0.27100000000000002},"min":{"1min":0.23100000000000001,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.40400000000000003,"5min":0.47199999999999998,"15min":0.47199999999999998},"last":0.27500000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051788,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:43:55 2023","interfaces":[{"interface":"back","average":{"1min":0.26100000000000001,"5min":0.26700000000000002,"15min":0.27000000000000002},"min":{"1min":0.17799999999999999,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.374,"5min":0.44,"15min":0.44},"last":0.30299999999999999},{"interface":"front","average":{"1min":0.28299999999999997,"5min":0.28100000000000003,"15min":0.28399999999999997},"min":{"1min":0.20899999999999999,"5min":0.17599999999999999,"15min":0.16600000000000001},"max":{"1min":0.40100000000000002,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.27800000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:44:28.923 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":1126,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":19,"modified":"2023-03-25T04:43:20.278959+0000","services":{}},"progress_events":{}}
, 2023-03-25T04:44:28.936 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now
, 2023-03-25T04:44:28.937 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace: 2023-03-25T04:44:48.786 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:44:48.800 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":606,"stamp":"2023-03-25T04:44:48.296805+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002182"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":41,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:42:34.382196+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:42:34.382196+0000","last_peered":"2023-03-25T04:42:34.382196+0000","last_clean":"2023-03-25T04:42:34.382196+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:42:34.382196+0000","last_undegraded":"2023-03-25T04:42:34.382196+0000","last_fullsized":"2023-03-25T04:42:34.382196+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:42:29.381744+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:42:29.381744+0000","last_peered":"2023-03-25T04:42:29.381744+0000","last_clean":"2023-03-25T04:42:29.381744+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:42:29.381744+0000","last_undegraded":"2023-03-25T04:42:29.381744+0000","last_fullsized":"2023-03-25T04:42:29.381744+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:23.375106+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:41:23.375106+0000","last_peered":"2023-03-25T04:41:23.375106+0000","last_clean":"2023-03-25T04:41:23.375106+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:41:23.375106+0000","last_undegraded":"2023-03-25T04:41:23.375106+0000","last_fullsized":"2023-03-25T04:41:23.375106+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:50.618930+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:41:50.618930+0000","last_peered":"2023-03-25T04:41:50.618930+0000","last_clean":"2023-03-25T04:41:50.618930+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:41:50.618930+0000","last_undegraded":"2023-03-25T04:41:50.618930+0000","last_fullsized":"2023-03-25T04:41:50.618930+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":41,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:58.377821+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:41:58.377821+0000","last_peered":"2023-03-25T04:41:58.377821+0000","last_clean":"2023-03-25T04:41:58.377821+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:41:58.377821+0000","last_undegraded":"2023-03-25T04:41:58.377821+0000","last_fullsized":"2023-03-25T04:41:58.377821+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:58.377836+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:41:58.377836+0000","last_peered":"2023-03-25T04:41:58.377836+0000","last_clean":"2023-03-25T04:41:58.377836+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:41:58.377836+0000","last_undegraded":"2023-03-25T04:41:58.377836+0000","last_fullsized":"2023-03-25T04:41:58.377836+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117206,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:44:01 2023","interfaces":[{"interface":"back","average":{"1min":0.27200000000000002,"5min":0.27000000000000002,"15min":0.26400000000000001},"min":{"1min":0.20399999999999999,"5min":0.17499999999999999,"15min":0.16},"max":{"1min":0.38400000000000001,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.216},{"interface":"front","average":{"1min":0.27600000000000002,"5min":0.27800000000000002,"15min":0.27100000000000002},"min":{"1min":0.23100000000000001,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.40400000000000003,"5min":0.47199999999999998,"15min":0.47199999999999998},"last":0.23999999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051796,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:43:55 2023","interfaces":[{"interface":"back","average":{"1min":0.26100000000000001,"5min":0.26700000000000002,"15min":0.27000000000000002},"min":{"1min":0.17799999999999999,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.374,"5min":0.44,"15min":0.44},"last":0.25600000000000001},{"interface":"front","average":{"1min":0.28299999999999997,"5min":0.28100000000000003,"15min":0.28399999999999997},"min":{"1min":0.20899999999999999,"5min":0.17599999999999999,"15min":0.16600000000000001},"max":{"1min":0.40100000000000002,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.28000000000000003}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:45:05.762 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:45:06.073 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all

Ignoring malparsed backtrace: 2023-03-25T04:45:25.658 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":624,"stamp":"2023-03-25T04:45:24.304294+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002366"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":41,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:42:34.382196+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:42:34.382196+0000","last_peered":"2023-03-25T04:42:34.382196+0000","last_clean":"2023-03-25T04:42:34.382196+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:42:34.382196+0000","last_undegraded":"2023-03-25T04:42:34.382196+0000","last_fullsized":"2023-03-25T04:42:34.382196+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:42:29.381744+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:42:29.381744+0000","last_peered":"2023-03-25T04:42:29.381744+0000","last_clean":"2023-03-25T04:42:29.381744+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:42:29.381744+0000","last_undegraded":"2023-03-25T04:42:29.381744+0000","last_fullsized":"2023-03-25T04:42:29.381744+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:23.375106+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:41:23.375106+0000","last_peered":"2023-03-25T04:41:23.375106+0000","last_clean":"2023-03-25T04:41:23.375106+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:41:23.375106+0000","last_undegraded":"2023-03-25T04:41:23.375106+0000","last_fullsized":"2023-03-25T04:41:23.375106+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:50.618930+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:41:50.618930+0000","last_peered":"2023-03-25T04:41:50.618930+0000","last_clean":"2023-03-25T04:41:50.618930+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:41:50.618930+0000","last_undegraded":"2023-03-25T04:41:50.618930+0000","last_fullsized":"2023-03-25T04:41:50.618930+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":41,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:58.377821+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:41:58.377821+0000","last_peered":"2023-03-25T04:41:58.377821+0000","last_clean":"2023-03-25T04:41:58.377821+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:41:58.377821+0000","last_undegraded":"2023-03-25T04:41:58.377821+0000","last_fullsized":"2023-03-25T04:41:58.377821+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:58.377836+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:41:58.377836+0000","last_peered":"2023-03-25T04:41:58.377836+0000","last_clean":"2023-03-25T04:41:58.377836+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:41:58.377836+0000","last_undegraded":"2023-03-25T04:41:58.377836+0000","last_fullsized":"2023-03-25T04:41:58.377836+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117213,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:45:01 2023","interfaces":[{"interface":"back","average":{"1min":0.251,"5min":0.26800000000000002,"15min":0.26400000000000001},"min":{"1min":0.183,"5min":0.17499999999999999,"15min":0.16},"max":{"1min":0.41499999999999998,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.26900000000000002},{"interface":"front","average":{"1min":0.28000000000000003,"5min":0.27900000000000003,"15min":0.27100000000000002},"min":{"1min":0.20999999999999999,"5min":0.185,"15min":0.17499999999999999},"max":{"1min":0.44400000000000001,"5min":0.47199999999999998,"15min":0.47199999999999998},"last":0.27900000000000003}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051803,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:44:55 2023","interfaces":[{"interface":"back","average":{"1min":0.252,"5min":0.26500000000000001,"15min":0.26900000000000002},"min":{"1min":0.183,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.39700000000000002,"5min":0.44,"15min":0.44},"last":0.20699999999999999},{"interface":"front","average":{"1min":0.27000000000000002,"5min":0.28100000000000003,"15min":0.28299999999999997},"min":{"1min":0.20300000000000001,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":0.42099999999999999,"5min":0.47599999999999998,"15min":0.48599999999999999},"last":0.23599999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:45:25.659 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:45:25.969 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:45:42.735+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:45:42.744+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:45:42.897 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace: 2023-03-25T04:46:00.183 INFO:teuthology.orchestra.run.smithi070.stdout:
, 2023-03-25T04:46:00.183 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":1217,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":20,"modified":"2023-03-25T04:45:42.307700+0000","services":{"mgr":{"daemons":{"summary":"","x":{"start_epoch":0,"start_stamp":"0.000000","gid":0,"addr":"(unrecognized address family 0)/0","metadata":{},"task_status":{}}}}}},"progress_events":{}}
, 2023-03-25T04:46:00.197 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now

Ignoring malparsed backtrace: 2023-03-25T04:46:19.910+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:46:20.039 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all

Ignoring malparsed backtrace: 2023-03-25T04:46:39.611 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}
, 2023-03-25T04:46:39.611 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json
, 2023-03-25T04:46:39.613+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:46:58.485+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:46:58.764+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:46:59.519+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:47:14.132 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:47:14.146 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":678,"stamp":"2023-03-25T04:47:12.325044+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002571"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":42,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:44.393426+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:44:44.393426+0000","last_peered":"2023-03-25T04:44:44.393426+0000","last_clean":"2023-03-25T04:44:44.393426+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:44:44.393426+0000","last_undegraded":"2023-03-25T04:44:44.393426+0000","last_fullsized":"2023-03-25T04:44:44.393426+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:39.393044+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:44:39.393044+0000","last_peered":"2023-03-25T04:44:39.393044+0000","last_clean":"2023-03-25T04:44:39.393044+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:44:39.393044+0000","last_undegraded":"2023-03-25T04:44:39.393044+0000","last_fullsized":"2023-03-25T04:44:39.393044+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:43:24.386177+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:43:24.386177+0000","last_peered":"2023-03-25T04:43:24.386177+0000","last_clean":"2023-03-25T04:43:24.386177+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:43:24.386177+0000","last_undegraded":"2023-03-25T04:43:24.386177+0000","last_fullsized":"2023-03-25T04:43:24.386177+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:43:55.629798+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:43:55.629798+0000","last_peered":"2023-03-25T04:43:55.629798+0000","last_clean":"2023-03-25T04:43:55.629798+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:43:55.629798+0000","last_undegraded":"2023-03-25T04:43:55.629798+0000","last_fullsized":"2023-03-25T04:43:55.629798+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":42,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:04.389904+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:44:04.389904+0000","last_peered":"2023-03-25T04:44:04.389904+0000","last_clean":"2023-03-25T04:44:04.389904+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:44:04.389904+0000","last_undegraded":"2023-03-25T04:44:04.389904+0000","last_fullsized":"2023-03-25T04:44:04.389904+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:04.389923+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:44:04.389923+0000","last_peered":"2023-03-25T04:44:04.389923+0000","last_clean":"2023-03-25T04:44:04.389923+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:44:04.389923+0000","last_undegraded":"2023-03-25T04:44:04.389923+0000","last_fullsized":"2023-03-25T04:44:04.389923+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117235,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:47:05 2023","interfaces":[{"interface":"back","average":{"1min":0.27800000000000002,"5min":0.27000000000000002,"15min":0.26300000000000001},"min":{"1min":0.20100000000000001,"5min":0.16900000000000001,"15min":0.16},"max":{"1min":0.39200000000000002,"5min":0.42999999999999999,"15min":0.45600000000000002},"last":0.379},{"interface":"front","average":{"1min":0.28399999999999997,"5min":0.28100000000000003,"15min":0.27200000000000002},"min":{"1min":0.17799999999999999,"5min":0.17799999999999999,"15min":0.17699999999999999},"max":{"1min":0.42599999999999999,"5min":0.47199999999999998,"15min":0.47199999999999998},"last":0.39800000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051825,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:47:01 2023","interfaces":[{"interface":"back","average":{"1min":0.27300000000000002,"5min":0.26100000000000001,"15min":0.27000000000000002},"min":{"1min":0.186,"5min":0.17399999999999999,"15min":0.17000000000000001},"max":{"1min":0.40400000000000003,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.23899999999999999},{"interface":"front","average":{"1min":0.28699999999999998,"5min":0.27500000000000002,"15min":0.28299999999999997},"min":{"1min":0.186,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.48599999999999999},"last":0.19600000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:47:31.198 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json
, 2023-03-25T04:47:31.411+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:47:31.789 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:47:49.509 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":696,"stamp":"2023-03-25T04:47:48.332093+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002697"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":42,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:44.393426+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:44:44.393426+0000","last_peered":"2023-03-25T04:44:44.393426+0000","last_clean":"2023-03-25T04:44:44.393426+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:44:44.393426+0000","last_undegraded":"2023-03-25T04:44:44.393426+0000","last_fullsized":"2023-03-25T04:44:44.393426+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:39.393044+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:44:39.393044+0000","last_peered":"2023-03-25T04:44:39.393044+0000","last_clean":"2023-03-25T04:44:39.393044+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:44:39.393044+0000","last_undegraded":"2023-03-25T04:44:39.393044+0000","last_fullsized":"2023-03-25T04:44:39.393044+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:45:24.396717+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:45:24.396717+0000","last_peered":"2023-03-25T04:45:24.396717+0000","last_clean":"2023-03-25T04:45:24.396717+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:45:24.396717+0000","last_undegraded":"2023-03-25T04:45:24.396717+0000","last_fullsized":"2023-03-25T04:45:24.396717+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:43:55.629798+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:43:55.629798+0000","last_peered":"2023-03-25T04:43:55.629798+0000","last_clean":"2023-03-25T04:43:55.629798+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:43:55.629798+0000","last_undegraded":"2023-03-25T04:43:55.629798+0000","last_fullsized":"2023-03-25T04:43:55.629798+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":42,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:04.389904+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:44:04.389904+0000","last_peered":"2023-03-25T04:44:04.389904+0000","last_clean":"2023-03-25T04:44:04.389904+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:44:04.389904+0000","last_undegraded":"2023-03-25T04:44:04.389904+0000","last_fullsized":"2023-03-25T04:44:04.389904+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:04.389923+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:44:04.389923+0000","last_peered":"2023-03-25T04:44:04.389923+0000","last_clean":"2023-03-25T04:44:04.389923+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:44:04.389923+0000","last_undegraded":"2023-03-25T04:44:04.389923+0000","last_fullsized":"2023-03-25T04:44:04.389923+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117242,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:47:05 2023","interfaces":[{"interface":"back","average":{"1min":0.27800000000000002,"5min":0.27000000000000002,"15min":0.26300000000000001},"min":{"1min":0.20100000000000001,"5min":0.16900000000000001,"15min":0.16},"max":{"1min":0.39200000000000002,"5min":0.42999999999999999,"15min":0.45600000000000002},"last":0.40500000000000003},{"interface":"front","average":{"1min":0.28399999999999997,"5min":0.28100000000000003,"15min":0.27200000000000002},"min":{"1min":0.17799999999999999,"5min":0.17799999999999999,"15min":0.17699999999999999},"max":{"1min":0.42599999999999999,"5min":0.47199999999999998,"15min":0.47199999999999998},"last":0.42399999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051834,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:47:01 2023","interfaces":[{"interface":"back","average":{"1min":0.27300000000000002,"5min":0.26100000000000001,"15min":0.27000000000000002},"min":{"1min":0.186,"5min":0.17399999999999999,"15min":0.17000000000000001},"max":{"1min":0.40400000000000003,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.25600000000000001},{"interface":"front","average":{"1min":0.28699999999999998,"5min":0.27500000000000002,"15min":0.28299999999999997},"min":{"1min":0.186,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.48599999999999999},"last":0.22800000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:47:49.510 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace:     ctx.managers[config['cluster']].wait_for_clean()
,   File "/home/teuthworker/src/github.com_ceph_ceph-c_511015c23eef0684a64171b710e432448df2b7c7/qa/tasks/ceph_manager.py", line 2747, in wait_for_clean
,     assert time.time() - start < timeout, \

Ignoring malparsed backtrace:  ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
,  1: /lib64/libpthread.so.0(+0x12cf0) [0x7f1453c2fcf0]
,  2: __pthread_mutex_lock()

Ignoring malparsed backtrace: 2023-03-25T04:28:05.172 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":142,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":5,"modified":"2023-03-25T04:27:12.092098+0000","services":{}},"progress_events":{}}
, 2023-03-25T04:28:05.185 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now
, 2023-03-25T04:28:05.186 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace: 2023-03-25T04:28:28.360 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":116,"stamp":"2023-03-25T04:28:28.108007+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002578"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.314661+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:27:45.314661+0000","last_peered":"2023-03-25T04:27:45.314661+0000","last_clean":"2023-03-25T04:27:45.314661+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:27:45.314661+0000","last_undegraded":"2023-03-25T04:27:45.314661+0000","last_fullsized":"2023-03-25T04:27:45.314661+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":25,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.310745+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:27:44.310745+0000","last_peered":"2023-03-25T04:27:44.310745+0000","last_clean":"2023-03-25T04:27:44.310745+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:27:44.310745+0000","last_undegraded":"2023-03-25T04:27:44.310745+0000","last_fullsized":"2023-03-25T04:27:44.310745+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":25,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.310650+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:27:44.310650+0000","last_peered":"2023-03-25T04:27:44.310650+0000","last_clean":"2023-03-25T04:27:44.310650+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:27:44.310650+0000","last_undegraded":"2023-03-25T04:27:44.310650+0000","last_fullsized":"2023-03-25T04:27:44.310650+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":26,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.312962+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:27:44.312962+0000","last_peered":"2023-03-25T04:27:44.312962+0000","last_clean":"2023-03-25T04:27:44.312962+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:27:44.312962+0000","last_undegraded":"2023-03-25T04:27:44.312962+0000","last_fullsized":"2023-03-25T04:27:44.312962+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315064+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:27:45.315064+0000","last_peered":"2023-03-25T04:27:45.315064+0000","last_clean":"2023-03-25T04:27:45.315064+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:27:45.315064+0000","last_undegraded":"2023-03-25T04:27:45.315064+0000","last_fullsized":"2023-03-25T04:27:45.315064+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":25,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.310707+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:27:44.310707+0000","last_peered":"2023-03-25T04:27:44.310707+0000","last_clean":"2023-03-25T04:27:44.310707+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:27:44.310707+0000","last_undegraded":"2023-03-25T04:27:44.310707+0000","last_fullsized":"2023-03-25T04:27:44.310707+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117007,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.375}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051597,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.186}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:28:28.362 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean
, 2023-03-25T04:28:28.362 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:28:45.655 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":124,"stamp":"2023-03-25T04:28:44.111265+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002492"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.314661+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:27:45.314661+0000","last_peered":"2023-03-25T04:27:45.314661+0000","last_clean":"2023-03-25T04:27:45.314661+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:27:45.314661+0000","last_undegraded":"2023-03-25T04:27:45.314661+0000","last_fullsized":"2023-03-25T04:27:45.314661+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":25,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.310745+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:27:44.310745+0000","last_peered":"2023-03-25T04:27:44.310745+0000","last_clean":"2023-03-25T04:27:44.310745+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:27:44.310745+0000","last_undegraded":"2023-03-25T04:27:44.310745+0000","last_fullsized":"2023-03-25T04:27:44.310745+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":25,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.310650+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:27:44.310650+0000","last_peered":"2023-03-25T04:27:44.310650+0000","last_clean":"2023-03-25T04:27:44.310650+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:27:44.310650+0000","last_undegraded":"2023-03-25T04:27:44.310650+0000","last_fullsized":"2023-03-25T04:27:44.310650+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":26,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.312962+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:27:44.312962+0000","last_peered":"2023-03-25T04:27:44.312962+0000","last_clean":"2023-03-25T04:27:44.312962+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:27:44.312962+0000","last_undegraded":"2023-03-25T04:27:44.312962+0000","last_fullsized":"2023-03-25T04:27:44.312962+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315064+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:27:45.315064+0000","last_peered":"2023-03-25T04:27:45.315064+0000","last_clean":"2023-03-25T04:27:45.315064+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:27:45.315064+0000","last_undegraded":"2023-03-25T04:27:45.315064+0000","last_fullsized":"2023-03-25T04:27:45.315064+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":25,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.310707+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:27:44.310707+0000","last_peered":"2023-03-25T04:27:44.310707+0000","last_clean":"2023-03-25T04:27:44.310707+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:27:44.310707+0000","last_undegraded":"2023-03-25T04:27:44.310707+0000","last_fullsized":"2023-03-25T04:27:44.310707+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117011,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:28:31 2023","interfaces":[{"interface":"back","average":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.36899999999999999},{"interface":"front","average":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"min":{"1min":0.19,"5min":0.19,"15min":0.19},"max":{"1min":0.46300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.39800000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051601,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:28:24 2023","interfaces":[{"interface":"back","average":{"1min":0.249,"5min":0.249,"15min":0.249},"min":{"1min":0.17699999999999999,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"last":0.254},{"interface":"front","average":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"min":{"1min":0.20200000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"last":0.312}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:28:45.656 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace: 2023-03-25T04:29:05.171 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}
, 2023-03-25T04:29:05.171 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json
, 2023-03-25T04:29:05.561 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:29:22.465 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:29:22.773 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all

Ignoring malparsed backtrace: 2023-03-25T04:29:42.405 INFO:teuthology.orchestra.run.smithi070.stdout:
, 2023-03-25T04:29:42.406 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":240,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":9,"modified":"2023-03-25T04:29:32.120358+0000","services":{"mon":{"daemons":{"summary":"","a":{"start_epoch":0,"start_stamp":"0.000000","gid":0,"addr":"(unrecognized address family 0)/0","metadata":{},"task_status":{}}}}}},"progress_events":{}}
, 2023-03-25T04:29:42.419 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:29:59.627 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":161,"stamp":"2023-03-25T04:29:58.125186+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002398"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315468+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:27:45.315468+0000","last_peered":"2023-03-25T04:27:45.315468+0000","last_clean":"2023-03-25T04:27:45.315468+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:27:45.315468+0000","last_undegraded":"2023-03-25T04:27:45.315468+0000","last_fullsized":"2023-03-25T04:27:45.315468+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312845+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:27:45.312845+0000","last_peered":"2023-03-25T04:27:45.312845+0000","last_clean":"2023-03-25T04:27:45.312845+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:27:45.312845+0000","last_undegraded":"2023-03-25T04:27:45.312845+0000","last_fullsized":"2023-03-25T04:27:45.312845+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312882+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:27:45.312882+0000","last_peered":"2023-03-25T04:27:45.312882+0000","last_clean":"2023-03-25T04:27:45.312882+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:27:45.312882+0000","last_undegraded":"2023-03-25T04:27:45.312882+0000","last_fullsized":"2023-03-25T04:27:45.312882+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.311730+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:27:45.311730+0000","last_peered":"2023-03-25T04:27:45.311730+0000","last_clean":"2023-03-25T04:27:45.311730+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:27:45.311730+0000","last_undegraded":"2023-03-25T04:27:45.311730+0000","last_fullsized":"2023-03-25T04:27:45.311730+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315735+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:27:45.315735+0000","last_peered":"2023-03-25T04:27:45.315735+0000","last_clean":"2023-03-25T04:27:45.315735+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:27:45.315735+0000","last_undegraded":"2023-03-25T04:27:45.315735+0000","last_fullsized":"2023-03-25T04:27:45.315735+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312833+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:27:45.312833+0000","last_peered":"2023-03-25T04:27:45.312833+0000","last_clean":"2023-03-25T04:27:45.312833+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:27:45.312833+0000","last_undegraded":"2023-03-25T04:27:45.312833+0000","last_fullsized":"2023-03-25T04:27:45.312833+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117026,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:29:33 2023","interfaces":[{"interface":"back","average":{"1min":0.26300000000000001,"5min":0.29099999999999998,"15min":0.29599999999999999},"min":{"1min":0.191,"5min":0.191,"15min":0.191},"max":{"1min":0.371,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.20499999999999999},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.29599999999999999,"15min":0.29999999999999999},"min":{"1min":0.21099999999999999,"5min":0.19,"15min":0.19},"max":{"1min":0.39800000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.19500000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051616,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:29:29 2023","interfaces":[{"interface":"back","average":{"1min":0.27200000000000002,"5min":0.253,"15min":0.25},"min":{"1min":0.183,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"last":0.26400000000000001},{"interface":"front","average":{"1min":0.28499999999999998,"5min":0.26800000000000002,"15min":0.26500000000000001},"min":{"1min":0.16600000000000001,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"last":0.29899999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:29:59.629 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean

Ignoring malparsed backtrace: 2023-03-25T04:30:16.889 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:30:16.903 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":170,"stamp":"2023-03-25T04:30:16.128427+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002222"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315468+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:27:45.315468+0000","last_peered":"2023-03-25T04:27:45.315468+0000","last_clean":"2023-03-25T04:27:45.315468+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:27:45.315468+0000","last_undegraded":"2023-03-25T04:27:45.315468+0000","last_fullsized":"2023-03-25T04:27:45.315468+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312845+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:27:45.312845+0000","last_peered":"2023-03-25T04:27:45.312845+0000","last_clean":"2023-03-25T04:27:45.312845+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:27:45.312845+0000","last_undegraded":"2023-03-25T04:27:45.312845+0000","last_fullsized":"2023-03-25T04:27:45.312845+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312882+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:27:45.312882+0000","last_peered":"2023-03-25T04:27:45.312882+0000","last_clean":"2023-03-25T04:27:45.312882+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:27:45.312882+0000","last_undegraded":"2023-03-25T04:27:45.312882+0000","last_fullsized":"2023-03-25T04:27:45.312882+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.311730+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:27:45.311730+0000","last_peered":"2023-03-25T04:27:45.311730+0000","last_clean":"2023-03-25T04:27:45.311730+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:27:45.311730+0000","last_undegraded":"2023-03-25T04:27:45.311730+0000","last_fullsized":"2023-03-25T04:27:45.311730+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315735+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:27:45.315735+0000","last_peered":"2023-03-25T04:27:45.315735+0000","last_clean":"2023-03-25T04:27:45.315735+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:27:45.315735+0000","last_undegraded":"2023-03-25T04:27:45.315735+0000","last_fullsized":"2023-03-25T04:27:45.315735+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312833+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:27:45.312833+0000","last_peered":"2023-03-25T04:27:45.312833+0000","last_clean":"2023-03-25T04:27:45.312833+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:27:45.312833+0000","last_undegraded":"2023-03-25T04:27:45.312833+0000","last_fullsized":"2023-03-25T04:27:45.312833+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117030,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:29:33 2023","interfaces":[{"interface":"back","average":{"1min":0.26300000000000001,"5min":0.29099999999999998,"15min":0.29599999999999999},"min":{"1min":0.191,"5min":0.191,"15min":0.191},"max":{"1min":0.371,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.245},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.29599999999999999,"15min":0.29999999999999999},"min":{"1min":0.21099999999999999,"5min":0.19,"15min":0.19},"max":{"1min":0.39800000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.20899999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051620,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:29:29 2023","interfaces":[{"interface":"back","average":{"1min":0.27200000000000002,"5min":0.253,"15min":0.25},"min":{"1min":0.183,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"last":0.19400000000000001},{"interface":"front","average":{"1min":0.28499999999999998,"5min":0.26800000000000002,"15min":0.26500000000000001},"min":{"1min":0.16600000000000001,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"last":0.23599999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:30:31.984+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:30:32.201+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:30:32.981+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)

Ignoring malparsed backtrace: 2023-03-25T04:30:48.449 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now
, 2023-03-25T04:30:48.450 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:30:48.760 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:31:08.342 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":196,"stamp":"2023-03-25T04:31:08.139018+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002431"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315468+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:27:45.315468+0000","last_peered":"2023-03-25T04:27:45.315468+0000","last_clean":"2023-03-25T04:27:45.315468+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:27:45.315468+0000","last_undegraded":"2023-03-25T04:27:45.315468+0000","last_fullsized":"2023-03-25T04:27:45.315468+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312845+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:27:45.312845+0000","last_peered":"2023-03-25T04:27:45.312845+0000","last_clean":"2023-03-25T04:27:45.312845+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:27:45.312845+0000","last_undegraded":"2023-03-25T04:27:45.312845+0000","last_fullsized":"2023-03-25T04:27:45.312845+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312882+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:27:45.312882+0000","last_peered":"2023-03-25T04:27:45.312882+0000","last_clean":"2023-03-25T04:27:45.312882+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:27:45.312882+0000","last_undegraded":"2023-03-25T04:27:45.312882+0000","last_fullsized":"2023-03-25T04:27:45.312882+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.311730+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:27:45.311730+0000","last_peered":"2023-03-25T04:27:45.311730+0000","last_clean":"2023-03-25T04:27:45.311730+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:27:45.311730+0000","last_undegraded":"2023-03-25T04:27:45.311730+0000","last_fullsized":"2023-03-25T04:27:45.311730+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315735+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:27:45.315735+0000","last_peered":"2023-03-25T04:27:45.315735+0000","last_clean":"2023-03-25T04:27:45.315735+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:27:45.315735+0000","last_undegraded":"2023-03-25T04:27:45.315735+0000","last_fullsized":"2023-03-25T04:27:45.315735+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312833+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:27:45.312833+0000","last_peered":"2023-03-25T04:27:45.312833+0000","last_clean":"2023-03-25T04:27:45.312833+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:27:45.312833+0000","last_undegraded":"2023-03-25T04:27:45.312833+0000","last_fullsized":"2023-03-25T04:27:45.312833+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117040,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:30:37 2023","interfaces":[{"interface":"back","average":{"1min":0.28799999999999998,"5min":0.28899999999999998,"15min":0.29499999999999998},"min":{"1min":0.20200000000000001,"5min":0.191,"15min":0.191},"max":{"1min":0.433,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.29799999999999999},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.28999999999999998,"15min":0.29799999999999999},"min":{"1min":0.17499999999999999,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":0.40300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.25700000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051630,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:30:32 2023","interfaces":[{"interface":"back","average":{"1min":0.25600000000000001,"5min":0.255,"15min":0.251},"min":{"1min":0.19400000000000001,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.35699999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"last":0.22800000000000001},{"interface":"front","average":{"1min":0.28699999999999998,"5min":0.27200000000000002,"15min":0.26600000000000001},"min":{"1min":0.20599999999999999,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.39400000000000002,"5min":0.42899999999999999,"15min":0.42899999999999999},"last":0.24099999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:31:08.343 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace: 2023-03-25T04:31:24.283+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:31:25.192+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:31:25.295+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)

Ignoring malparsed backtrace: 2023-03-25T04:31:40.257 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:31:40.271 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":212,"stamp":"2023-03-25T04:31:40.144732+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002208"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315468+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:27:45.315468+0000","last_peered":"2023-03-25T04:27:45.315468+0000","last_clean":"2023-03-25T04:27:45.315468+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:27:45.315468+0000","last_undegraded":"2023-03-25T04:27:45.315468+0000","last_fullsized":"2023-03-25T04:27:45.315468+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312845+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:27:45.312845+0000","last_peered":"2023-03-25T04:27:45.312845+0000","last_clean":"2023-03-25T04:27:45.312845+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:27:45.312845+0000","last_undegraded":"2023-03-25T04:27:45.312845+0000","last_fullsized":"2023-03-25T04:27:45.312845+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:23.308607+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:29:23.308607+0000","last_peered":"2023-03-25T04:29:23.308607+0000","last_clean":"2023-03-25T04:29:23.308607+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:29:23.308607+0000","last_undegraded":"2023-03-25T04:29:23.308607+0000","last_fullsized":"2023-03-25T04:29:23.308607+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":28,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:20.554710+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:29:20.554710+0000","last_peered":"2023-03-25T04:29:20.554710+0000","last_clean":"2023-03-25T04:29:20.554710+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:29:20.554710+0000","last_undegraded":"2023-03-25T04:29:20.554710+0000","last_fullsized":"2023-03-25T04:29:20.554710+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309216+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:29:28.309216+0000","last_peered":"2023-03-25T04:29:28.309216+0000","last_clean":"2023-03-25T04:29:28.309216+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:29:28.309216+0000","last_undegraded":"2023-03-25T04:29:28.309216+0000","last_fullsized":"2023-03-25T04:29:28.309216+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309233+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:29:28.309233+0000","last_peered":"2023-03-25T04:29:28.309233+0000","last_clean":"2023-03-25T04:29:28.309233+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:29:28.309233+0000","last_undegraded":"2023-03-25T04:29:28.309233+0000","last_fullsized":"2023-03-25T04:29:28.309233+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117046,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:30:37 2023","interfaces":[{"interface":"back","average":{"1min":0.28799999999999998,"5min":0.28899999999999998,"15min":0.29499999999999998},"min":{"1min":0.20200000000000001,"5min":0.191,"15min":0.191},"max":{"1min":0.433,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.216},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.28999999999999998,"15min":0.29799999999999999},"min":{"1min":0.17499999999999999,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":0.40300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.24299999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051637,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:31:35 2023","interfaces":[{"interface":"back","average":{"1min":0.26100000000000001,"5min":0.25700000000000001,"15min":0.251},"min":{"1min":0.19,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.35899999999999999,"5min":0.41599999999999998,"15min":0.41599999999999998},"last":0.221},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.27400000000000002,"15min":0.26700000000000002},"min":{"1min":0.20200000000000001,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.374,"5min":0.42899999999999999,"15min":0.42899999999999999},"last":0.24099999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:32:01.906+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:32:02.388+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:32:02.794 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:32:22.326 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":233,"stamp":"2023-03-25T04:32:22.153289+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002545"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:33.309748+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:29:33.309748+0000","last_peered":"2023-03-25T04:29:33.309748+0000","last_clean":"2023-03-25T04:29:33.309748+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:29:33.309748+0000","last_undegraded":"2023-03-25T04:29:33.309748+0000","last_fullsized":"2023-03-25T04:29:33.309748+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:33.309729+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:29:33.309729+0000","last_peered":"2023-03-25T04:29:33.309729+0000","last_clean":"2023-03-25T04:29:33.309729+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:29:33.309729+0000","last_undegraded":"2023-03-25T04:29:33.309729+0000","last_fullsized":"2023-03-25T04:29:33.309729+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:23.308607+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:29:23.308607+0000","last_peered":"2023-03-25T04:29:23.308607+0000","last_clean":"2023-03-25T04:29:23.308607+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:29:23.308607+0000","last_undegraded":"2023-03-25T04:29:23.308607+0000","last_fullsized":"2023-03-25T04:29:23.308607+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":28,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:20.554710+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:29:20.554710+0000","last_peered":"2023-03-25T04:29:20.554710+0000","last_clean":"2023-03-25T04:29:20.554710+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:29:20.554710+0000","last_undegraded":"2023-03-25T04:29:20.554710+0000","last_fullsized":"2023-03-25T04:29:20.554710+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309216+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:29:28.309216+0000","last_peered":"2023-03-25T04:29:28.309216+0000","last_clean":"2023-03-25T04:29:28.309216+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:29:28.309216+0000","last_undegraded":"2023-03-25T04:29:28.309216+0000","last_fullsized":"2023-03-25T04:29:28.309216+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309233+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:29:28.309233+0000","last_peered":"2023-03-25T04:29:28.309233+0000","last_clean":"2023-03-25T04:29:28.309233+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:29:28.309233+0000","last_undegraded":"2023-03-25T04:29:28.309233+0000","last_fullsized":"2023-03-25T04:29:28.309233+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117055,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:31:42 2023","interfaces":[{"interface":"back","average":{"1min":0.27500000000000002,"5min":0.28399999999999997,"15min":0.29399999999999998},"min":{"1min":0.19800000000000001,"5min":0.191,"15min":0.191},"max":{"1min":0.36899999999999999,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.32300000000000001},{"interface":"front","average":{"1min":0.27900000000000003,"5min":0.28499999999999998,"15min":0.29599999999999999},"min":{"1min":0.216,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":0.40799999999999997,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.27100000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051646,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:31:35 2023","interfaces":[{"interface":"back","average":{"1min":0.26100000000000001,"5min":0.25700000000000001,"15min":0.251},"min":{"1min":0.19,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.35899999999999999,"5min":0.41599999999999998,"15min":0.41599999999999998},"last":0.25600000000000001},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.27400000000000002,"15min":0.26700000000000002},"min":{"1min":0.20200000000000001,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.374,"5min":0.42899999999999999,"15min":0.42899999999999999},"last":0.29199999999999998}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:32:22.327 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean
, 2023-03-25T04:32:22.327 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}

Ignoring malparsed backtrace: 2023-03-25T04:32:39.836 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:32:39.849 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":241,"stamp":"2023-03-25T04:32:38.156312+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002346"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:33.309748+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:29:33.309748+0000","last_peered":"2023-03-25T04:29:33.309748+0000","last_clean":"2023-03-25T04:29:33.309748+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:29:33.309748+0000","last_undegraded":"2023-03-25T04:29:33.309748+0000","last_fullsized":"2023-03-25T04:29:33.309748+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:33.309729+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:29:33.309729+0000","last_peered":"2023-03-25T04:29:33.309729+0000","last_clean":"2023-03-25T04:29:33.309729+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:29:33.309729+0000","last_undegraded":"2023-03-25T04:29:33.309729+0000","last_fullsized":"2023-03-25T04:29:33.309729+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:23.308607+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:29:23.308607+0000","last_peered":"2023-03-25T04:29:23.308607+0000","last_clean":"2023-03-25T04:29:23.308607+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:29:23.308607+0000","last_undegraded":"2023-03-25T04:29:23.308607+0000","last_fullsized":"2023-03-25T04:29:23.308607+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":28,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:20.554710+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:29:20.554710+0000","last_peered":"2023-03-25T04:29:20.554710+0000","last_clean":"2023-03-25T04:29:20.554710+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:29:20.554710+0000","last_undegraded":"2023-03-25T04:29:20.554710+0000","last_fullsized":"2023-03-25T04:29:20.554710+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309216+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:29:28.309216+0000","last_peered":"2023-03-25T04:29:28.309216+0000","last_clean":"2023-03-25T04:29:28.309216+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:29:28.309216+0000","last_undegraded":"2023-03-25T04:29:28.309216+0000","last_fullsized":"2023-03-25T04:29:28.309216+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309233+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:29:28.309233+0000","last_peered":"2023-03-25T04:29:28.309233+0000","last_clean":"2023-03-25T04:29:28.309233+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:29:28.309233+0000","last_undegraded":"2023-03-25T04:29:28.309233+0000","last_fullsized":"2023-03-25T04:29:28.309233+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117058,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:31:42 2023","interfaces":[{"interface":"back","average":{"1min":0.27500000000000002,"5min":0.28399999999999997,"15min":0.29399999999999998},"min":{"1min":0.19800000000000001,"5min":0.191,"15min":0.191},"max":{"1min":0.36899999999999999,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.30599999999999999},{"interface":"front","average":{"1min":0.27900000000000003,"5min":0.28499999999999998,"15min":0.29599999999999999},"min":{"1min":0.216,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":0.40799999999999997,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.32500000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051649,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:31:35 2023","interfaces":[{"interface":"back","average":{"1min":0.26100000000000001,"5min":0.25700000000000001,"15min":0.251},"min":{"1min":0.19,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.35899999999999999,"5min":0.41599999999999998,"15min":0.41599999999999998},"last":0.24199999999999999},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.27400000000000002,"15min":0.26700000000000002},"min":{"1min":0.20200000000000001,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.374,"5min":0.42899999999999999,"15min":0.42899999999999999},"last":0.22600000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:32:54.944+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:32:55.469+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:32:55.927+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)

Ignoring malparsed backtrace: 2023-03-25T04:33:13.958 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean
, 2023-03-25T04:33:13.959 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}
, 2023-03-25T04:33:13.959 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json

Ignoring malparsed backtrace: 2023-03-25T04:33:31.199 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":267,"stamp":"2023-03-25T04:33:30.166660+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002269"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:33.309748+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:29:33.309748+0000","last_peered":"2023-03-25T04:29:33.309748+0000","last_clean":"2023-03-25T04:29:33.309748+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:29:33.309748+0000","last_undegraded":"2023-03-25T04:29:33.309748+0000","last_fullsized":"2023-03-25T04:29:33.309748+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:33.309729+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:29:33.309729+0000","last_peered":"2023-03-25T04:29:33.309729+0000","last_clean":"2023-03-25T04:29:33.309729+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:29:33.309729+0000","last_undegraded":"2023-03-25T04:29:33.309729+0000","last_fullsized":"2023-03-25T04:29:33.309729+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":28,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:31:23.320894+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:31:23.320894+0000","last_peered":"2023-03-25T04:31:23.320894+0000","last_clean":"2023-03-25T04:31:23.320894+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:31:23.320894+0000","last_undegraded":"2023-03-25T04:31:23.320894+0000","last_fullsized":"2023-03-25T04:31:23.320894+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":28,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:20.554710+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:29:20.554710+0000","last_peered":"2023-03-25T04:29:20.554710+0000","last_clean":"2023-03-25T04:29:20.554710+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:29:20.554710+0000","last_undegraded":"2023-03-25T04:29:20.554710+0000","last_fullsized":"2023-03-25T04:29:20.554710+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309216+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:29:28.309216+0000","last_peered":"2023-03-25T04:29:28.309216+0000","last_clean":"2023-03-25T04:29:28.309216+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:29:28.309216+0000","last_undegraded":"2023-03-25T04:29:28.309216+0000","last_fullsized":"2023-03-25T04:29:28.309216+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309233+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:29:28.309233+0000","last_peered":"2023-03-25T04:29:28.309233+0000","last_clean":"2023-03-25T04:29:28.309233+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:29:28.309233+0000","last_undegraded":"2023-03-25T04:29:28.309233+0000","last_fullsized":"2023-03-25T04:29:28.309233+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117068,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:32:46 2023","interfaces":[{"interface":"back","average":{"1min":0.27000000000000002,"5min":0.27900000000000003,"15min":0.29199999999999998},"min":{"1min":0.17899999999999999,"5min":0.17899999999999999,"15min":0.17899999999999999},"max":{"1min":0.35599999999999998,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.28799999999999998},{"interface":"front","average":{"1min":0.26800000000000002,"5min":0.27800000000000002,"15min":0.29399999999999998},"min":{"1min":0.19900000000000001,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":0.377,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.315}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051660,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:32:36 2023","interfaces":[{"interface":"back","average":{"1min":0.248,"5min":0.25700000000000001,"15min":0.251},"min":{"1min":0.17000000000000001,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"last":0.22700000000000001},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.27600000000000002,"15min":0.26800000000000002},"min":{"1min":0.192,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"last":0.20399999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:33:31.201 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:33:31.400+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)

Ignoring malparsed backtrace: 2023-03-25T04:33:47.531+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:33:47.951+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:33:48.505+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)

Ignoring malparsed backtrace: 2023-03-25T04:34:06.104 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json
, 2023-03-25T04:34:06.118+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:34:06.497 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:34:23.399 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:34:23.665+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:34:23.712 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:34:40.195+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:34:40.523+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:34:40.676 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace: 2023-03-25T04:35:03.381+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:35:04.139+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:35:04.391+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)

Ignoring malparsed backtrace: 2023-03-25T04:35:22.738 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean
, 2023-03-25T04:35:22.739 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}
, 2023-03-25T04:35:22.739 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json

Ignoring malparsed backtrace: 2023-03-25T04:35:41.393+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:35:42.352+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:35:42.376+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:35:57.384 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":340,"stamp":"2023-03-25T04:35:56.194992+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.001930"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":36,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:31:43.322894+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:31:43.322894+0000","last_peered":"2023-03-25T04:31:43.322894+0000","last_clean":"2023-03-25T04:31:43.322894+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:31:43.322894+0000","last_undegraded":"2023-03-25T04:31:43.322894+0000","last_fullsized":"2023-03-25T04:31:43.322894+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":28,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:31:43.322876+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:31:43.322876+0000","last_peered":"2023-03-25T04:31:43.322876+0000","last_clean":"2023-03-25T04:31:43.322876+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:31:43.322876+0000","last_undegraded":"2023-03-25T04:31:43.322876+0000","last_fullsized":"2023-03-25T04:31:43.322876+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:23.331998+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:33:23.331998+0000","last_peered":"2023-03-25T04:33:23.331998+0000","last_clean":"2023-03-25T04:33:23.331998+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:33:23.331998+0000","last_undegraded":"2023-03-25T04:33:23.331998+0000","last_fullsized":"2023-03-25T04:33:23.331998+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:30.576723+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:33:30.576723+0000","last_peered":"2023-03-25T04:33:30.576723+0000","last_clean":"2023-03-25T04:33:30.576723+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:33:30.576723+0000","last_undegraded":"2023-03-25T04:33:30.576723+0000","last_fullsized":"2023-03-25T04:33:30.576723+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":37,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:38.333447+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:33:38.333447+0000","last_peered":"2023-03-25T04:33:38.333447+0000","last_clean":"2023-03-25T04:33:38.333447+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:33:38.333447+0000","last_undegraded":"2023-03-25T04:33:38.333447+0000","last_fullsized":"2023-03-25T04:33:38.333447+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:38.333467+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:33:38.333467+0000","last_peered":"2023-03-25T04:33:38.333467+0000","last_clean":"2023-03-25T04:33:38.333467+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:33:38.333467+0000","last_undegraded":"2023-03-25T04:33:38.333467+0000","last_fullsized":"2023-03-25T04:33:38.333467+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117098,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:35:49 2023","interfaces":[{"interface":"back","average":{"1min":0.255,"5min":0.26000000000000001,"15min":0.28199999999999997},"min":{"1min":0.17899999999999999,"5min":0.16,"15min":0.16},"max":{"1min":0.36399999999999999,"5min":0.42599999999999999,"15min":0.45100000000000001},"last":0.26100000000000001},{"interface":"front","average":{"1min":0.26000000000000001,"5min":0.26700000000000002,"15min":0.28599999999999998},"min":{"1min":0.188,"5min":0.17999999999999999,"15min":0.17499999999999999},"max":{"1min":0.39100000000000001,"5min":0.45200000000000001,"15min":0.46300000000000002},"last":0.28399999999999997}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051689,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:35:42 2023","interfaces":[{"interface":"back","average":{"1min":0.29399999999999998,"5min":0.27200000000000002,"15min":0.25800000000000001},"min":{"1min":0.17299999999999999,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.41099999999999998,"5min":0.42699999999999999,"15min":0.42699999999999999},"last":0.20399999999999999},{"interface":"front","average":{"1min":0.30299999999999999,"5min":0.28399999999999997,"15min":0.27300000000000002},"min":{"1min":0.19700000000000001,"5min":0.192,"15min":0.16600000000000001},"max":{"1min":0.43099999999999999,"5min":0.45800000000000002,"15min":0.45800000000000002},"last":0.22}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:35:57.624+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)

Ignoring malparsed backtrace: 2023-03-25T04:36:17.226 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:36:17.239 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":350,"stamp":"2023-03-25T04:36:16.198866+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002148"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":37,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:53.335026+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:33:53.335026+0000","last_peered":"2023-03-25T04:33:53.335026+0000","last_clean":"2023-03-25T04:33:53.335026+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:33:53.335026+0000","last_undegraded":"2023-03-25T04:33:53.335026+0000","last_fullsized":"2023-03-25T04:33:53.335026+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:53.335009+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:33:53.335009+0000","last_peered":"2023-03-25T04:33:53.335009+0000","last_clean":"2023-03-25T04:33:53.335009+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:33:53.335009+0000","last_undegraded":"2023-03-25T04:33:53.335009+0000","last_fullsized":"2023-03-25T04:33:53.335009+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:23.331998+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:33:23.331998+0000","last_peered":"2023-03-25T04:33:23.331998+0000","last_clean":"2023-03-25T04:33:23.331998+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:33:23.331998+0000","last_undegraded":"2023-03-25T04:33:23.331998+0000","last_fullsized":"2023-03-25T04:33:23.331998+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:30.576723+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:33:30.576723+0000","last_peered":"2023-03-25T04:33:30.576723+0000","last_clean":"2023-03-25T04:33:30.576723+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:33:30.576723+0000","last_undegraded":"2023-03-25T04:33:30.576723+0000","last_fullsized":"2023-03-25T04:33:30.576723+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":37,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:38.333447+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:33:38.333447+0000","last_peered":"2023-03-25T04:33:38.333447+0000","last_clean":"2023-03-25T04:33:38.333447+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:33:38.333447+0000","last_undegraded":"2023-03-25T04:33:38.333447+0000","last_fullsized":"2023-03-25T04:33:38.333447+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:38.333467+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:33:38.333467+0000","last_peered":"2023-03-25T04:33:38.333467+0000","last_clean":"2023-03-25T04:33:38.333467+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:33:38.333467+0000","last_undegraded":"2023-03-25T04:33:38.333467+0000","last_fullsized":"2023-03-25T04:33:38.333467+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117102,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:35:49 2023","interfaces":[{"interface":"back","average":{"1min":0.255,"5min":0.26000000000000001,"15min":0.28199999999999997},"min":{"1min":0.17899999999999999,"5min":0.16,"15min":0.16},"max":{"1min":0.36399999999999999,"5min":0.42599999999999999,"15min":0.45100000000000001},"last":0.28399999999999997},{"interface":"front","average":{"1min":0.26000000000000001,"5min":0.26700000000000002,"15min":0.28599999999999998},"min":{"1min":0.188,"5min":0.17999999999999999,"15min":0.17499999999999999},"max":{"1min":0.39100000000000001,"5min":0.45200000000000001,"15min":0.46300000000000002},"last":0.30299999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051693,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:35:42 2023","interfaces":[{"interface":"back","average":{"1min":0.29399999999999998,"5min":0.27200000000000002,"15min":0.25800000000000001},"min":{"1min":0.17299999999999999,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.41099999999999998,"5min":0.42699999999999999,"15min":0.42699999999999999},"last":0.35099999999999998},{"interface":"front","average":{"1min":0.30299999999999999,"5min":0.28399999999999997,"15min":0.27300000000000002},"min":{"1min":0.19700000000000001,"5min":0.192,"15min":0.16600000000000001},"max":{"1min":0.43099999999999999,"5min":0.45800000000000002,"15min":0.45800000000000002},"last":0.32900000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:36:34.556+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:36:34.717+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:36:35.583+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)

Ignoring malparsed backtrace: 2023-03-25T04:36:52.680 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json
, 2023-03-25T04:36:52.701+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:36:53.067 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:37:09.963 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:37:10.273 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all

Ignoring malparsed backtrace: 2023-03-25T04:37:26.856+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:37:27.288 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:37:27.538+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:37:41.903 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":392,"stamp":"2023-03-25T04:37:40.213373+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002012"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":37,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:53.335026+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:33:53.335026+0000","last_peered":"2023-03-25T04:33:53.335026+0000","last_clean":"2023-03-25T04:33:53.335026+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:33:53.335026+0000","last_undegraded":"2023-03-25T04:33:53.335026+0000","last_fullsized":"2023-03-25T04:33:53.335026+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:53.335009+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:33:53.335009+0000","last_peered":"2023-03-25T04:33:53.335009+0000","last_clean":"2023-03-25T04:33:53.335009+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:33:53.335009+0000","last_undegraded":"2023-03-25T04:33:53.335009+0000","last_fullsized":"2023-03-25T04:33:53.335009+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:23.343585+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:35:23.343585+0000","last_peered":"2023-03-25T04:35:23.343585+0000","last_clean":"2023-03-25T04:35:23.343585+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:35:23.343585+0000","last_undegraded":"2023-03-25T04:35:23.343585+0000","last_fullsized":"2023-03-25T04:35:23.343585+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:30.576723+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:33:30.576723+0000","last_peered":"2023-03-25T04:33:30.576723+0000","last_clean":"2023-03-25T04:33:30.576723+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:33:30.576723+0000","last_undegraded":"2023-03-25T04:33:30.576723+0000","last_fullsized":"2023-03-25T04:33:30.576723+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":37,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:38.333447+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:33:38.333447+0000","last_peered":"2023-03-25T04:33:38.333447+0000","last_clean":"2023-03-25T04:33:38.333447+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:33:38.333447+0000","last_undegraded":"2023-03-25T04:33:38.333447+0000","last_fullsized":"2023-03-25T04:33:38.333447+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:38.333467+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:33:38.333467+0000","last_peered":"2023-03-25T04:33:38.333467+0000","last_clean":"2023-03-25T04:33:38.333467+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:33:38.333467+0000","last_undegraded":"2023-03-25T04:33:38.333467+0000","last_fullsized":"2023-03-25T04:33:38.333467+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117119,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:36:50 2023","interfaces":[{"interface":"back","average":{"1min":0.254,"5min":0.25600000000000001,"15min":0.27900000000000003},"min":{"1min":0.20499999999999999,"5min":0.16,"15min":0.16},"max":{"1min":0.371,"5min":0.42599999999999999,"15min":0.45100000000000001},"last":0.23000000000000001},{"interface":"front","average":{"1min":0.26800000000000002,"5min":0.26500000000000001,"15min":0.28399999999999997},"min":{"1min":0.19700000000000001,"5min":0.17999999999999999,"15min":0.17499999999999999},"max":{"1min":0.34699999999999998,"5min":0.45200000000000001,"15min":0.46300000000000002},"last":0.20899999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051710,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:36:44 2023","interfaces":[{"interface":"back","average":{"1min":0.28399999999999997,"5min":0.27700000000000002,"15min":0.26100000000000001},"min":{"1min":0.192,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.40899999999999997,"5min":0.42699999999999999,"15min":0.42699999999999999},"last":0.39700000000000002},{"interface":"front","average":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.27500000000000002},"min":{"1min":0.182,"5min":0.182,"15min":0.16600000000000001},"max":{"1min":0.439,"5min":0.45800000000000002,"15min":0.45800000000000002},"last":0.439}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:37:42.400+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)

Ignoring malparsed backtrace: 2023-03-25T04:38:04.766+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:38:04.922 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:38:05.246 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:38:19.605 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":411,"stamp":"2023-03-25T04:38:18.220966+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002368"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":38,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:36:03.347070+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:36:03.347070+0000","last_peered":"2023-03-25T04:36:03.347070+0000","last_clean":"2023-03-25T04:36:03.347070+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:36:03.347070+0000","last_undegraded":"2023-03-25T04:36:03.347070+0000","last_fullsized":"2023-03-25T04:36:03.347070+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:36:03.347025+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:36:03.347025+0000","last_peered":"2023-03-25T04:36:03.347025+0000","last_clean":"2023-03-25T04:36:03.347025+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:36:03.347025+0000","last_undegraded":"2023-03-25T04:36:03.347025+0000","last_fullsized":"2023-03-25T04:36:03.347025+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:23.343585+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:35:23.343585+0000","last_peered":"2023-03-25T04:35:23.343585+0000","last_clean":"2023-03-25T04:35:23.343585+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:35:23.343585+0000","last_undegraded":"2023-03-25T04:35:23.343585+0000","last_fullsized":"2023-03-25T04:35:23.343585+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:35.587025+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:35:35.587025+0000","last_peered":"2023-03-25T04:35:35.587025+0000","last_clean":"2023-03-25T04:35:35.587025+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:35:35.587025+0000","last_undegraded":"2023-03-25T04:35:35.587025+0000","last_fullsized":"2023-03-25T04:35:35.587025+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":38,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:43.345308+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:35:43.345308+0000","last_peered":"2023-03-25T04:35:43.345308+0000","last_clean":"2023-03-25T04:35:43.345308+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:35:43.345308+0000","last_undegraded":"2023-03-25T04:35:43.345308+0000","last_fullsized":"2023-03-25T04:35:43.345308+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:43.345324+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:35:43.345324+0000","last_peered":"2023-03-25T04:35:43.345324+0000","last_clean":"2023-03-25T04:35:43.345324+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:35:43.345324+0000","last_undegraded":"2023-03-25T04:35:43.345324+0000","last_fullsized":"2023-03-25T04:35:43.345324+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117127,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:37:53 2023","interfaces":[{"interface":"back","average":{"1min":0.25800000000000001,"5min":0.253,"15min":0.27700000000000002},"min":{"1min":0.184,"5min":0.16,"15min":0.16},"max":{"1min":0.44600000000000001,"5min":0.44600000000000001,"15min":0.45100000000000001},"last":0.35199999999999998},{"interface":"front","average":{"1min":0.26900000000000002,"5min":0.26500000000000001,"15min":0.28199999999999997},"min":{"1min":0.192,"5min":0.17999999999999999,"15min":0.17499999999999999},"max":{"1min":0.46400000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"last":0.33200000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051717,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:37:44 2023","interfaces":[{"interface":"back","average":{"1min":0.27300000000000002,"5min":0.28199999999999997,"15min":0.26200000000000001},"min":{"1min":0.17899999999999999,"5min":0.17299999999999999,"15min":0.17000000000000001},"max":{"1min":0.41799999999999998,"5min":0.42099999999999999,"15min":0.42699999999999999},"last":0.28699999999999998},{"interface":"front","average":{"1min":0.29399999999999998,"5min":0.29099999999999998,"15min":0.27700000000000002},"min":{"1min":0.19900000000000001,"5min":0.182,"15min":0.16600000000000001},"max":{"1min":0.441,"5min":0.441,"15min":0.45800000000000002},"last":0.26300000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:38:19.718+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:38:20.315+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)

Ignoring malparsed backtrace: 2023-03-25T04:38:44.283 INFO:teuthology.orchestra.run.smithi070.stdout:
, 2023-03-25T04:38:44.283 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":781,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":16,"modified":"2023-03-25T04:38:40.225321+0000","services":{}},"progress_events":{}}
, 2023-03-25T04:38:44.296 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now

Ignoring malparsed backtrace: 2023-03-25T04:39:02.477+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:39:02.578+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:39:03.524+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:39:20.359 INFO:teuthology.orchestra.run.smithi070.stdout:
, 2023-03-25T04:39:20.359 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":818,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":16,"modified":"2023-03-25T04:38:40.225321+0000","services":{}},"progress_events":{}}
, 2023-03-25T04:39:20.372 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now

Ignoring malparsed backtrace: 2023-03-25T04:39:37.527 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:39:37.541 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":450,"stamp":"2023-03-25T04:39:36.235332+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002086"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":38,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:36:03.347070+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:36:03.347070+0000","last_peered":"2023-03-25T04:36:03.347070+0000","last_clean":"2023-03-25T04:36:03.347070+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:36:03.347070+0000","last_undegraded":"2023-03-25T04:36:03.347070+0000","last_fullsized":"2023-03-25T04:36:03.347070+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:36:03.347025+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:36:03.347025+0000","last_peered":"2023-03-25T04:36:03.347025+0000","last_clean":"2023-03-25T04:36:03.347025+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:36:03.347025+0000","last_undegraded":"2023-03-25T04:36:03.347025+0000","last_fullsized":"2023-03-25T04:36:03.347025+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:23.353778+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:37:23.353778+0000","last_peered":"2023-03-25T04:37:23.353778+0000","last_clean":"2023-03-25T04:37:23.353778+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:37:23.353778+0000","last_undegraded":"2023-03-25T04:37:23.353778+0000","last_fullsized":"2023-03-25T04:37:23.353778+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:35.587025+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:35:35.587025+0000","last_peered":"2023-03-25T04:35:35.587025+0000","last_clean":"2023-03-25T04:35:35.587025+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:35:35.587025+0000","last_undegraded":"2023-03-25T04:35:35.587025+0000","last_fullsized":"2023-03-25T04:35:35.587025+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":38,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:43.345308+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:35:43.345308+0000","last_peered":"2023-03-25T04:35:43.345308+0000","last_clean":"2023-03-25T04:35:43.345308+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:35:43.345308+0000","last_undegraded":"2023-03-25T04:35:43.345308+0000","last_fullsized":"2023-03-25T04:35:43.345308+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:43.345324+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:35:43.345324+0000","last_peered":"2023-03-25T04:35:43.345324+0000","last_clean":"2023-03-25T04:35:43.345324+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:35:43.345324+0000","last_undegraded":"2023-03-25T04:35:43.345324+0000","last_fullsized":"2023-03-25T04:35:43.345324+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117143,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:38:54 2023","interfaces":[{"interface":"back","average":{"1min":0.25800000000000001,"5min":0.25700000000000001,"15min":0.27400000000000002},"min":{"1min":0.192,"5min":0.17399999999999999,"15min":0.16},"max":{"1min":0.39400000000000002,"5min":0.44600000000000001,"15min":0.45100000000000001},"last":0.26500000000000001},{"interface":"front","average":{"1min":0.25900000000000001,"5min":0.26700000000000002,"15min":0.27900000000000003},"min":{"1min":0.19500000000000001,"5min":0.188,"15min":0.17499999999999999},"max":{"1min":0.41499999999999998,"5min":0.46400000000000002,"15min":0.46400000000000002},"last":0.29099999999999998}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051733,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:38:45 2023","interfaces":[{"interface":"back","average":{"1min":0.27300000000000002,"5min":0.28100000000000003,"15min":0.26400000000000001},"min":{"1min":0.19400000000000001,"5min":0.17299999999999999,"15min":0.17000000000000001},"max":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"last":0.27300000000000002},{"interface":"front","average":{"1min":0.28799999999999998,"5min":0.29099999999999998,"15min":0.27800000000000002},"min":{"1min":0.19800000000000001,"5min":0.182,"15min":0.16600000000000001},"max":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"last":0.308}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:39:54.416+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:39:54.455 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:39:54.780 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:40:09.288+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:40:09.389+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:40:10.267+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)

Ignoring malparsed backtrace: 2023-03-25T04:40:28.984 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":476,"stamp":"2023-03-25T04:40:28.245911+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002693"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":39,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:38:13.358968+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:38:13.358968+0000","last_peered":"2023-03-25T04:38:13.358968+0000","last_clean":"2023-03-25T04:38:13.358968+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:38:13.358968+0000","last_undegraded":"2023-03-25T04:38:13.358968+0000","last_fullsized":"2023-03-25T04:38:13.358968+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:38:13.358922+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:38:13.358922+0000","last_peered":"2023-03-25T04:38:13.358922+0000","last_clean":"2023-03-25T04:38:13.358922+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:38:13.358922+0000","last_undegraded":"2023-03-25T04:38:13.358922+0000","last_fullsized":"2023-03-25T04:38:13.358922+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:23.353778+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:37:23.353778+0000","last_peered":"2023-03-25T04:37:23.353778+0000","last_clean":"2023-03-25T04:37:23.353778+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:37:23.353778+0000","last_undegraded":"2023-03-25T04:37:23.353778+0000","last_fullsized":"2023-03-25T04:37:23.353778+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:40.598579+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:37:40.598579+0000","last_peered":"2023-03-25T04:37:40.598579+0000","last_clean":"2023-03-25T04:37:40.598579+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:37:40.598579+0000","last_undegraded":"2023-03-25T04:37:40.598579+0000","last_fullsized":"2023-03-25T04:37:40.598579+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":39,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:48.356344+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:37:48.356344+0000","last_peered":"2023-03-25T04:37:48.356344+0000","last_clean":"2023-03-25T04:37:48.356344+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:37:48.356344+0000","last_undegraded":"2023-03-25T04:37:48.356344+0000","last_fullsized":"2023-03-25T04:37:48.356344+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:48.356365+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:37:48.356365+0000","last_peered":"2023-03-25T04:37:48.356365+0000","last_clean":"2023-03-25T04:37:48.356365+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:37:48.356365+0000","last_undegraded":"2023-03-25T04:37:48.356365+0000","last_fullsized":"2023-03-25T04:37:48.356365+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117153,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:39:54 2023","interfaces":[{"interface":"back","average":{"1min":0.25900000000000001,"5min":0.25600000000000001,"15min":0.27100000000000002},"min":{"1min":0.188,"5min":0.17899999999999999,"15min":0.16},"max":{"1min":0.41999999999999998,"5min":0.44600000000000001,"15min":0.45100000000000001},"last":0.255},{"interface":"front","average":{"1min":0.27500000000000002,"5min":0.26600000000000001,"15min":0.27700000000000002},"min":{"1min":0.17699999999999999,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.45600000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"last":0.26800000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051743,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:39:47 2023","interfaces":[{"interface":"back","average":{"1min":0.26300000000000001,"5min":0.27700000000000002,"15min":0.26500000000000001},"min":{"1min":0.187,"5min":0.17299999999999999,"15min":0.17000000000000001},"max":{"1min":0.433,"5min":0.433,"15min":0.433},"last":0.36599999999999999},{"interface":"front","average":{"1min":0.27100000000000002,"5min":0.28799999999999998,"15min":0.27900000000000003},"min":{"1min":0.20100000000000001,"5min":0.182,"15min":0.16600000000000001},"max":{"1min":0.48599999999999999,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.34999999999999998}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:40:28.986 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean
, 2023-03-25T04:40:28.986 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}

Ignoring malparsed backtrace: 2023-03-25T04:40:47.229+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:40:47.503+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:40:48.251+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)

Ignoring malparsed backtrace: 2023-03-25T04:41:03.267 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:41:03.578 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all

Ignoring malparsed backtrace: 2023-03-25T04:41:21.347 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":502,"stamp":"2023-03-25T04:41:20.256204+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002342"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":39,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:38:13.358968+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:38:13.358968+0000","last_peered":"2023-03-25T04:38:13.358968+0000","last_clean":"2023-03-25T04:38:13.358968+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:38:13.358968+0000","last_undegraded":"2023-03-25T04:38:13.358968+0000","last_fullsized":"2023-03-25T04:38:13.358968+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:38:13.358922+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:38:13.358922+0000","last_peered":"2023-03-25T04:38:13.358922+0000","last_clean":"2023-03-25T04:38:13.358922+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:38:13.358922+0000","last_undegraded":"2023-03-25T04:38:13.358922+0000","last_fullsized":"2023-03-25T04:38:13.358922+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:23.353778+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:37:23.353778+0000","last_peered":"2023-03-25T04:37:23.353778+0000","last_clean":"2023-03-25T04:37:23.353778+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:37:23.353778+0000","last_undegraded":"2023-03-25T04:37:23.353778+0000","last_fullsized":"2023-03-25T04:37:23.353778+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:40.598579+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:37:40.598579+0000","last_peered":"2023-03-25T04:37:40.598579+0000","last_clean":"2023-03-25T04:37:40.598579+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:37:40.598579+0000","last_undegraded":"2023-03-25T04:37:40.598579+0000","last_fullsized":"2023-03-25T04:37:40.598579+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":39,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:48.356344+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:37:48.356344+0000","last_peered":"2023-03-25T04:37:48.356344+0000","last_clean":"2023-03-25T04:37:48.356344+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:37:48.356344+0000","last_undegraded":"2023-03-25T04:37:48.356344+0000","last_fullsized":"2023-03-25T04:37:48.356344+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:48.356365+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:37:48.356365+0000","last_peered":"2023-03-25T04:37:48.356365+0000","last_clean":"2023-03-25T04:37:48.356365+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:37:48.356365+0000","last_undegraded":"2023-03-25T04:37:48.356365+0000","last_fullsized":"2023-03-25T04:37:48.356365+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117163,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:40:56 2023","interfaces":[{"interface":"back","average":{"1min":0.28799999999999998,"5min":0.26300000000000001,"15min":0.27100000000000002},"min":{"1min":0.188,"5min":0.184,"15min":0.16},"max":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.17499999999999999},{"interface":"front","average":{"1min":0.29599999999999999,"5min":0.27300000000000002,"15min":0.27700000000000002},"min":{"1min":0.20599999999999999,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.43099999999999999,"5min":0.46400000000000002,"15min":0.46400000000000002},"last":0.20699999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051754,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:40:52 2023","interfaces":[{"interface":"back","average":{"1min":0.28599999999999998,"5min":0.27500000000000002,"15min":0.26700000000000002},"min":{"1min":0.17899999999999999,"5min":0.17899999999999999,"15min":0.17000000000000001},"max":{"1min":0.44,"5min":0.44,"15min":0.44},"last":0.23699999999999999},{"interface":"front","average":{"1min":0.29799999999999999,"5min":0.28699999999999998,"15min":0.28100000000000003},"min":{"1min":0.20300000000000001,"5min":0.182,"15min":0.16600000000000001},"max":{"1min":0.47599999999999998,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.28000000000000003}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:41:21.349 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean
, 2023-03-25T04:41:21.349 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}

Ignoring malparsed backtrace: 2023-03-25T04:41:39.867+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:41:40.383+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:41:40.905+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:41:58.219 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json
, 2023-03-25T04:41:58.319+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:41:58.644 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:42:18.193 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":530,"stamp":"2023-03-25T04:42:16.266659+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002356"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":39,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:38:13.358968+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:38:13.358968+0000","last_peered":"2023-03-25T04:38:13.358968+0000","last_clean":"2023-03-25T04:38:13.358968+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:38:13.358968+0000","last_undegraded":"2023-03-25T04:38:13.358968+0000","last_fullsized":"2023-03-25T04:38:13.358968+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:38:13.358922+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:38:13.358922+0000","last_peered":"2023-03-25T04:38:13.358922+0000","last_clean":"2023-03-25T04:38:13.358922+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:38:13.358922+0000","last_undegraded":"2023-03-25T04:38:13.358922+0000","last_fullsized":"2023-03-25T04:38:13.358922+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:23.364593+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:39:23.364593+0000","last_peered":"2023-03-25T04:39:23.364593+0000","last_clean":"2023-03-25T04:39:23.364593+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:39:23.364593+0000","last_undegraded":"2023-03-25T04:39:23.364593+0000","last_fullsized":"2023-03-25T04:39:23.364593+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:45.608475+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:39:45.608475+0000","last_peered":"2023-03-25T04:39:45.608475+0000","last_clean":"2023-03-25T04:39:45.608475+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:39:45.608475+0000","last_undegraded":"2023-03-25T04:39:45.608475+0000","last_fullsized":"2023-03-25T04:39:45.608475+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":40,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:53.367398+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:39:53.367398+0000","last_peered":"2023-03-25T04:39:53.367398+0000","last_clean":"2023-03-25T04:39:53.367398+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:39:53.367398+0000","last_undegraded":"2023-03-25T04:39:53.367398+0000","last_fullsized":"2023-03-25T04:39:53.367398+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:53.367424+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:39:53.367424+0000","last_peered":"2023-03-25T04:39:53.367424+0000","last_clean":"2023-03-25T04:39:53.367424+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:39:53.367424+0000","last_undegraded":"2023-03-25T04:39:53.367424+0000","last_fullsized":"2023-03-25T04:39:53.367424+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117175,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:41:58 2023","interfaces":[{"interface":"back","average":{"1min":0.25800000000000001,"5min":0.26400000000000001,"15min":0.26800000000000002},"min":{"1min":0.17499999999999999,"5min":0.17499999999999999,"15min":0.16},"max":{"1min":0.39100000000000001,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.22900000000000001},{"interface":"front","average":{"1min":0.26300000000000001,"5min":0.27200000000000002,"15min":0.27400000000000002},"min":{"1min":0.20599999999999999,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.35699999999999998,"5min":0.46400000000000002,"15min":0.46400000000000002},"last":0.26400000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051765,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:41:52 2023","interfaces":[{"interface":"back","average":{"1min":0.26500000000000001,"5min":0.27200000000000002,"15min":0.26800000000000002},"min":{"1min":0.17000000000000001,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.39200000000000002,"5min":0.44,"15min":0.44},"last":0.23899999999999999},{"interface":"front","average":{"1min":0.28100000000000003,"5min":0.28599999999999998,"15min":0.28199999999999997},"min":{"1min":0.19400000000000001,"5min":0.19400000000000001,"15min":0.16600000000000001},"max":{"1min":0.41399999999999998,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.254}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:42:18.297+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:42:18.958+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:42:35.512 INFO:teuthology.orchestra.run.smithi070.stdout:
, 2023-03-25T04:42:35.512 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":1013,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":17,"modified":"2023-03-25T04:41:54.262744+0000","services":{}},"progress_events":{}}
, 2023-03-25T04:42:35.525 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:42:52.741 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":548,"stamp":"2023-03-25T04:42:52.273601+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002221"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":40,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:40:23.370056+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:40:23.370056+0000","last_peered":"2023-03-25T04:40:23.370056+0000","last_clean":"2023-03-25T04:40:23.370056+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:40:23.370056+0000","last_undegraded":"2023-03-25T04:40:23.370056+0000","last_fullsized":"2023-03-25T04:40:23.370056+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:40:23.370039+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:40:23.370039+0000","last_peered":"2023-03-25T04:40:23.370039+0000","last_clean":"2023-03-25T04:40:23.370039+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:40:23.370039+0000","last_undegraded":"2023-03-25T04:40:23.370039+0000","last_fullsized":"2023-03-25T04:40:23.370039+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:23.364593+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:39:23.364593+0000","last_peered":"2023-03-25T04:39:23.364593+0000","last_clean":"2023-03-25T04:39:23.364593+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:39:23.364593+0000","last_undegraded":"2023-03-25T04:39:23.364593+0000","last_fullsized":"2023-03-25T04:39:23.364593+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:45.608475+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:39:45.608475+0000","last_peered":"2023-03-25T04:39:45.608475+0000","last_clean":"2023-03-25T04:39:45.608475+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:39:45.608475+0000","last_undegraded":"2023-03-25T04:39:45.608475+0000","last_fullsized":"2023-03-25T04:39:45.608475+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":40,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:53.367398+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:39:53.367398+0000","last_peered":"2023-03-25T04:39:53.367398+0000","last_clean":"2023-03-25T04:39:53.367398+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:39:53.367398+0000","last_undegraded":"2023-03-25T04:39:53.367398+0000","last_fullsized":"2023-03-25T04:39:53.367398+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:53.367424+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:39:53.367424+0000","last_peered":"2023-03-25T04:39:53.367424+0000","last_clean":"2023-03-25T04:39:53.367424+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:39:53.367424+0000","last_undegraded":"2023-03-25T04:39:53.367424+0000","last_fullsized":"2023-03-25T04:39:53.367424+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117182,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:41:58 2023","interfaces":[{"interface":"back","average":{"1min":0.25800000000000001,"5min":0.26400000000000001,"15min":0.26800000000000002},"min":{"1min":0.17499999999999999,"5min":0.17499999999999999,"15min":0.16},"max":{"1min":0.39100000000000001,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.19600000000000001},{"interface":"front","average":{"1min":0.26300000000000001,"5min":0.27200000000000002,"15min":0.27400000000000002},"min":{"1min":0.20599999999999999,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.35699999999999998,"5min":0.46400000000000002,"15min":0.46400000000000002},"last":0.219}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051773,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:41:52 2023","interfaces":[{"interface":"back","average":{"1min":0.26500000000000001,"5min":0.27200000000000002,"15min":0.26800000000000002},"min":{"1min":0.17000000000000001,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.39200000000000002,"5min":0.44,"15min":0.44},"last":0.19},{"interface":"front","average":{"1min":0.28100000000000003,"5min":0.28599999999999998,"15min":0.28199999999999997},"min":{"1min":0.19400000000000001,"5min":0.19400000000000001,"15min":0.16600000000000001},"max":{"1min":0.41399999999999998,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.17599999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:42:52.742 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean

Ignoring malparsed backtrace: 2023-03-25T04:43:09.973 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:43:09.986 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":556,"stamp":"2023-03-25T04:43:08.276556+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002359"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":40,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:40:23.370056+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:40:23.370056+0000","last_peered":"2023-03-25T04:40:23.370056+0000","last_clean":"2023-03-25T04:40:23.370056+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:40:23.370056+0000","last_undegraded":"2023-03-25T04:40:23.370056+0000","last_fullsized":"2023-03-25T04:40:23.370056+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:40:23.370039+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:40:23.370039+0000","last_peered":"2023-03-25T04:40:23.370039+0000","last_clean":"2023-03-25T04:40:23.370039+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:40:23.370039+0000","last_undegraded":"2023-03-25T04:40:23.370039+0000","last_fullsized":"2023-03-25T04:40:23.370039+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:23.364593+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:39:23.364593+0000","last_peered":"2023-03-25T04:39:23.364593+0000","last_clean":"2023-03-25T04:39:23.364593+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:39:23.364593+0000","last_undegraded":"2023-03-25T04:39:23.364593+0000","last_fullsized":"2023-03-25T04:39:23.364593+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:45.608475+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:39:45.608475+0000","last_peered":"2023-03-25T04:39:45.608475+0000","last_clean":"2023-03-25T04:39:45.608475+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:39:45.608475+0000","last_undegraded":"2023-03-25T04:39:45.608475+0000","last_fullsized":"2023-03-25T04:39:45.608475+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":40,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:53.367398+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:39:53.367398+0000","last_peered":"2023-03-25T04:39:53.367398+0000","last_clean":"2023-03-25T04:39:53.367398+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:39:53.367398+0000","last_undegraded":"2023-03-25T04:39:53.367398+0000","last_fullsized":"2023-03-25T04:39:53.367398+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:53.367424+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:39:53.367424+0000","last_peered":"2023-03-25T04:39:53.367424+0000","last_clean":"2023-03-25T04:39:53.367424+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:39:53.367424+0000","last_undegraded":"2023-03-25T04:39:53.367424+0000","last_fullsized":"2023-03-25T04:39:53.367424+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117185,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:43:00 2023","interfaces":[{"interface":"back","average":{"1min":0.27400000000000002,"5min":0.26700000000000002,"15min":0.26600000000000001},"min":{"1min":0.19600000000000001,"5min":0.17499999999999999,"15min":0.16},"max":{"1min":0.42999999999999999,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.27800000000000002},{"interface":"front","average":{"1min":0.28199999999999997,"5min":0.27500000000000002,"15min":0.27300000000000002},"min":{"1min":0.185,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.47199999999999998,"5min":0.47199999999999998,"15min":0.47199999999999998},"last":0.247}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051776,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:42:54 2023","interfaces":[{"interface":"back","average":{"1min":0.26300000000000001,"5min":0.27000000000000002,"15min":0.26900000000000002},"min":{"1min":0.17399999999999999,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.438,"5min":0.44,"15min":0.44},"last":0.20100000000000001},{"interface":"front","average":{"1min":0.27400000000000002,"5min":0.28199999999999997,"15min":0.28299999999999997},"min":{"1min":0.17599999999999999,"5min":0.17599999999999999,"15min":0.16600000000000001},"max":{"1min":0.41599999999999998,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.23499999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:43:25.042+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:43:25.411+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:43:26.013+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:43:47.937+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:43:48.534+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:43:48.889+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:44:09.318 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:44:09.331 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":586,"stamp":"2023-03-25T04:44:08.288933+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002697"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":40,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:40:23.370056+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:40:23.370056+0000","last_peered":"2023-03-25T04:40:23.370056+0000","last_clean":"2023-03-25T04:40:23.370056+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:40:23.370056+0000","last_undegraded":"2023-03-25T04:40:23.370056+0000","last_fullsized":"2023-03-25T04:40:23.370056+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:40:23.370039+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:40:23.370039+0000","last_peered":"2023-03-25T04:40:23.370039+0000","last_clean":"2023-03-25T04:40:23.370039+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:40:23.370039+0000","last_undegraded":"2023-03-25T04:40:23.370039+0000","last_fullsized":"2023-03-25T04:40:23.370039+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:23.375106+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:41:23.375106+0000","last_peered":"2023-03-25T04:41:23.375106+0000","last_clean":"2023-03-25T04:41:23.375106+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:41:23.375106+0000","last_undegraded":"2023-03-25T04:41:23.375106+0000","last_fullsized":"2023-03-25T04:41:23.375106+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:50.618930+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:41:50.618930+0000","last_peered":"2023-03-25T04:41:50.618930+0000","last_clean":"2023-03-25T04:41:50.618930+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:41:50.618930+0000","last_undegraded":"2023-03-25T04:41:50.618930+0000","last_fullsized":"2023-03-25T04:41:50.618930+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":41,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:58.377821+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:41:58.377821+0000","last_peered":"2023-03-25T04:41:58.377821+0000","last_clean":"2023-03-25T04:41:58.377821+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:41:58.377821+0000","last_undegraded":"2023-03-25T04:41:58.377821+0000","last_fullsized":"2023-03-25T04:41:58.377821+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:58.377836+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:41:58.377836+0000","last_peered":"2023-03-25T04:41:58.377836+0000","last_clean":"2023-03-25T04:41:58.377836+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:41:58.377836+0000","last_undegraded":"2023-03-25T04:41:58.377836+0000","last_fullsized":"2023-03-25T04:41:58.377836+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117197,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:44:01 2023","interfaces":[{"interface":"back","average":{"1min":0.27200000000000002,"5min":0.27000000000000002,"15min":0.26400000000000001},"min":{"1min":0.20399999999999999,"5min":0.17499999999999999,"15min":0.16},"max":{"1min":0.38400000000000001,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.245},{"interface":"front","average":{"1min":0.27600000000000002,"5min":0.27800000000000002,"15min":0.27100000000000002},"min":{"1min":0.23100000000000001,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.40400000000000003,"5min":0.47199999999999998,"15min":0.47199999999999998},"last":0.27500000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051788,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:43:55 2023","interfaces":[{"interface":"back","average":{"1min":0.26100000000000001,"5min":0.26700000000000002,"15min":0.27000000000000002},"min":{"1min":0.17799999999999999,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.374,"5min":0.44,"15min":0.44},"last":0.30299999999999999},{"interface":"front","average":{"1min":0.28299999999999997,"5min":0.28100000000000003,"15min":0.28399999999999997},"min":{"1min":0.20899999999999999,"5min":0.17599999999999999,"15min":0.16600000000000001},"max":{"1min":0.40100000000000002,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.27800000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:44:28.923 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":1126,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":19,"modified":"2023-03-25T04:43:20.278959+0000","services":{}},"progress_events":{}}
, 2023-03-25T04:44:28.936 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now
, 2023-03-25T04:44:28.937 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace: 2023-03-25T04:44:48.786 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:44:48.800 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":606,"stamp":"2023-03-25T04:44:48.296805+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002182"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":41,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:42:34.382196+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:42:34.382196+0000","last_peered":"2023-03-25T04:42:34.382196+0000","last_clean":"2023-03-25T04:42:34.382196+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:42:34.382196+0000","last_undegraded":"2023-03-25T04:42:34.382196+0000","last_fullsized":"2023-03-25T04:42:34.382196+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:42:29.381744+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:42:29.381744+0000","last_peered":"2023-03-25T04:42:29.381744+0000","last_clean":"2023-03-25T04:42:29.381744+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:42:29.381744+0000","last_undegraded":"2023-03-25T04:42:29.381744+0000","last_fullsized":"2023-03-25T04:42:29.381744+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:23.375106+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:41:23.375106+0000","last_peered":"2023-03-25T04:41:23.375106+0000","last_clean":"2023-03-25T04:41:23.375106+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:41:23.375106+0000","last_undegraded":"2023-03-25T04:41:23.375106+0000","last_fullsized":"2023-03-25T04:41:23.375106+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:50.618930+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:41:50.618930+0000","last_peered":"2023-03-25T04:41:50.618930+0000","last_clean":"2023-03-25T04:41:50.618930+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:41:50.618930+0000","last_undegraded":"2023-03-25T04:41:50.618930+0000","last_fullsized":"2023-03-25T04:41:50.618930+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":41,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:58.377821+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:41:58.377821+0000","last_peered":"2023-03-25T04:41:58.377821+0000","last_clean":"2023-03-25T04:41:58.377821+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:41:58.377821+0000","last_undegraded":"2023-03-25T04:41:58.377821+0000","last_fullsized":"2023-03-25T04:41:58.377821+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:58.377836+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:41:58.377836+0000","last_peered":"2023-03-25T04:41:58.377836+0000","last_clean":"2023-03-25T04:41:58.377836+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:41:58.377836+0000","last_undegraded":"2023-03-25T04:41:58.377836+0000","last_fullsized":"2023-03-25T04:41:58.377836+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117206,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:44:01 2023","interfaces":[{"interface":"back","average":{"1min":0.27200000000000002,"5min":0.27000000000000002,"15min":0.26400000000000001},"min":{"1min":0.20399999999999999,"5min":0.17499999999999999,"15min":0.16},"max":{"1min":0.38400000000000001,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.216},{"interface":"front","average":{"1min":0.27600000000000002,"5min":0.27800000000000002,"15min":0.27100000000000002},"min":{"1min":0.23100000000000001,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.40400000000000003,"5min":0.47199999999999998,"15min":0.47199999999999998},"last":0.23999999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051796,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:43:55 2023","interfaces":[{"interface":"back","average":{"1min":0.26100000000000001,"5min":0.26700000000000002,"15min":0.27000000000000002},"min":{"1min":0.17799999999999999,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.374,"5min":0.44,"15min":0.44},"last":0.25600000000000001},{"interface":"front","average":{"1min":0.28299999999999997,"5min":0.28100000000000003,"15min":0.28399999999999997},"min":{"1min":0.20899999999999999,"5min":0.17599999999999999,"15min":0.16600000000000001},"max":{"1min":0.40100000000000002,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.28000000000000003}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:45:05.762 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:45:06.073 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all

Ignoring malparsed backtrace: 2023-03-25T04:45:25.658 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":624,"stamp":"2023-03-25T04:45:24.304294+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002366"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":41,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:42:34.382196+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:42:34.382196+0000","last_peered":"2023-03-25T04:42:34.382196+0000","last_clean":"2023-03-25T04:42:34.382196+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:42:34.382196+0000","last_undegraded":"2023-03-25T04:42:34.382196+0000","last_fullsized":"2023-03-25T04:42:34.382196+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:42:29.381744+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:42:29.381744+0000","last_peered":"2023-03-25T04:42:29.381744+0000","last_clean":"2023-03-25T04:42:29.381744+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:42:29.381744+0000","last_undegraded":"2023-03-25T04:42:29.381744+0000","last_fullsized":"2023-03-25T04:42:29.381744+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:23.375106+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:41:23.375106+0000","last_peered":"2023-03-25T04:41:23.375106+0000","last_clean":"2023-03-25T04:41:23.375106+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:41:23.375106+0000","last_undegraded":"2023-03-25T04:41:23.375106+0000","last_fullsized":"2023-03-25T04:41:23.375106+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:50.618930+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:41:50.618930+0000","last_peered":"2023-03-25T04:41:50.618930+0000","last_clean":"2023-03-25T04:41:50.618930+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:41:50.618930+0000","last_undegraded":"2023-03-25T04:41:50.618930+0000","last_fullsized":"2023-03-25T04:41:50.618930+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":41,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:58.377821+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:41:58.377821+0000","last_peered":"2023-03-25T04:41:58.377821+0000","last_clean":"2023-03-25T04:41:58.377821+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:41:58.377821+0000","last_undegraded":"2023-03-25T04:41:58.377821+0000","last_fullsized":"2023-03-25T04:41:58.377821+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:58.377836+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:41:58.377836+0000","last_peered":"2023-03-25T04:41:58.377836+0000","last_clean":"2023-03-25T04:41:58.377836+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:41:58.377836+0000","last_undegraded":"2023-03-25T04:41:58.377836+0000","last_fullsized":"2023-03-25T04:41:58.377836+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117213,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:45:01 2023","interfaces":[{"interface":"back","average":{"1min":0.251,"5min":0.26800000000000002,"15min":0.26400000000000001},"min":{"1min":0.183,"5min":0.17499999999999999,"15min":0.16},"max":{"1min":0.41499999999999998,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.26900000000000002},{"interface":"front","average":{"1min":0.28000000000000003,"5min":0.27900000000000003,"15min":0.27100000000000002},"min":{"1min":0.20999999999999999,"5min":0.185,"15min":0.17499999999999999},"max":{"1min":0.44400000000000001,"5min":0.47199999999999998,"15min":0.47199999999999998},"last":0.27900000000000003}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051803,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:44:55 2023","interfaces":[{"interface":"back","average":{"1min":0.252,"5min":0.26500000000000001,"15min":0.26900000000000002},"min":{"1min":0.183,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.39700000000000002,"5min":0.44,"15min":0.44},"last":0.20699999999999999},{"interface":"front","average":{"1min":0.27000000000000002,"5min":0.28100000000000003,"15min":0.28299999999999997},"min":{"1min":0.20300000000000001,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":0.42099999999999999,"5min":0.47599999999999998,"15min":0.48599999999999999},"last":0.23599999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:45:25.659 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:45:25.969 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:45:42.735+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:45:42.744+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:45:42.897 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace: 2023-03-25T04:46:00.183 INFO:teuthology.orchestra.run.smithi070.stdout:
, 2023-03-25T04:46:00.183 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":1217,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":20,"modified":"2023-03-25T04:45:42.307700+0000","services":{"mgr":{"daemons":{"summary":"","x":{"start_epoch":0,"start_stamp":"0.000000","gid":0,"addr":"(unrecognized address family 0)/0","metadata":{},"task_status":{}}}}}},"progress_events":{}}
, 2023-03-25T04:46:00.197 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now

Ignoring malparsed backtrace: 2023-03-25T04:46:19.910+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:46:20.039 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all

Ignoring malparsed backtrace: 2023-03-25T04:46:39.611 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}
, 2023-03-25T04:46:39.611 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json
, 2023-03-25T04:46:39.613+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:46:58.485+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:46:58.764+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:46:59.519+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:47:14.132 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:47:14.146 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":678,"stamp":"2023-03-25T04:47:12.325044+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002571"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":42,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:44.393426+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:44:44.393426+0000","last_peered":"2023-03-25T04:44:44.393426+0000","last_clean":"2023-03-25T04:44:44.393426+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:44:44.393426+0000","last_undegraded":"2023-03-25T04:44:44.393426+0000","last_fullsized":"2023-03-25T04:44:44.393426+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:39.393044+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:44:39.393044+0000","last_peered":"2023-03-25T04:44:39.393044+0000","last_clean":"2023-03-25T04:44:39.393044+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:44:39.393044+0000","last_undegraded":"2023-03-25T04:44:39.393044+0000","last_fullsized":"2023-03-25T04:44:39.393044+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:43:24.386177+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:43:24.386177+0000","last_peered":"2023-03-25T04:43:24.386177+0000","last_clean":"2023-03-25T04:43:24.386177+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:43:24.386177+0000","last_undegraded":"2023-03-25T04:43:24.386177+0000","last_fullsized":"2023-03-25T04:43:24.386177+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:43:55.629798+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:43:55.629798+0000","last_peered":"2023-03-25T04:43:55.629798+0000","last_clean":"2023-03-25T04:43:55.629798+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:43:55.629798+0000","last_undegraded":"2023-03-25T04:43:55.629798+0000","last_fullsized":"2023-03-25T04:43:55.629798+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":42,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:04.389904+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:44:04.389904+0000","last_peered":"2023-03-25T04:44:04.389904+0000","last_clean":"2023-03-25T04:44:04.389904+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:44:04.389904+0000","last_undegraded":"2023-03-25T04:44:04.389904+0000","last_fullsized":"2023-03-25T04:44:04.389904+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:04.389923+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:44:04.389923+0000","last_peered":"2023-03-25T04:44:04.389923+0000","last_clean":"2023-03-25T04:44:04.389923+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:44:04.389923+0000","last_undegraded":"2023-03-25T04:44:04.389923+0000","last_fullsized":"2023-03-25T04:44:04.389923+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117235,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:47:05 2023","interfaces":[{"interface":"back","average":{"1min":0.27800000000000002,"5min":0.27000000000000002,"15min":0.26300000000000001},"min":{"1min":0.20100000000000001,"5min":0.16900000000000001,"15min":0.16},"max":{"1min":0.39200000000000002,"5min":0.42999999999999999,"15min":0.45600000000000002},"last":0.379},{"interface":"front","average":{"1min":0.28399999999999997,"5min":0.28100000000000003,"15min":0.27200000000000002},"min":{"1min":0.17799999999999999,"5min":0.17799999999999999,"15min":0.17699999999999999},"max":{"1min":0.42599999999999999,"5min":0.47199999999999998,"15min":0.47199999999999998},"last":0.39800000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051825,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:47:01 2023","interfaces":[{"interface":"back","average":{"1min":0.27300000000000002,"5min":0.26100000000000001,"15min":0.27000000000000002},"min":{"1min":0.186,"5min":0.17399999999999999,"15min":0.17000000000000001},"max":{"1min":0.40400000000000003,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.23899999999999999},{"interface":"front","average":{"1min":0.28699999999999998,"5min":0.27500000000000002,"15min":0.28299999999999997},"min":{"1min":0.186,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.48599999999999999},"last":0.19600000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:47:31.198 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json
, 2023-03-25T04:47:31.411+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:47:31.789 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:47:49.509 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":696,"stamp":"2023-03-25T04:47:48.332093+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002697"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":42,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:44.393426+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:44:44.393426+0000","last_peered":"2023-03-25T04:44:44.393426+0000","last_clean":"2023-03-25T04:44:44.393426+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:44:44.393426+0000","last_undegraded":"2023-03-25T04:44:44.393426+0000","last_fullsized":"2023-03-25T04:44:44.393426+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:39.393044+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:44:39.393044+0000","last_peered":"2023-03-25T04:44:39.393044+0000","last_clean":"2023-03-25T04:44:39.393044+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:44:39.393044+0000","last_undegraded":"2023-03-25T04:44:39.393044+0000","last_fullsized":"2023-03-25T04:44:39.393044+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:45:24.396717+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:45:24.396717+0000","last_peered":"2023-03-25T04:45:24.396717+0000","last_clean":"2023-03-25T04:45:24.396717+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:45:24.396717+0000","last_undegraded":"2023-03-25T04:45:24.396717+0000","last_fullsized":"2023-03-25T04:45:24.396717+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:43:55.629798+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:43:55.629798+0000","last_peered":"2023-03-25T04:43:55.629798+0000","last_clean":"2023-03-25T04:43:55.629798+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:43:55.629798+0000","last_undegraded":"2023-03-25T04:43:55.629798+0000","last_fullsized":"2023-03-25T04:43:55.629798+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":42,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:04.389904+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:44:04.389904+0000","last_peered":"2023-03-25T04:44:04.389904+0000","last_clean":"2023-03-25T04:44:04.389904+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:44:04.389904+0000","last_undegraded":"2023-03-25T04:44:04.389904+0000","last_fullsized":"2023-03-25T04:44:04.389904+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:04.389923+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:44:04.389923+0000","last_peered":"2023-03-25T04:44:04.389923+0000","last_clean":"2023-03-25T04:44:04.389923+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:44:04.389923+0000","last_undegraded":"2023-03-25T04:44:04.389923+0000","last_fullsized":"2023-03-25T04:44:04.389923+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117242,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:47:05 2023","interfaces":[{"interface":"back","average":{"1min":0.27800000000000002,"5min":0.27000000000000002,"15min":0.26300000000000001},"min":{"1min":0.20100000000000001,"5min":0.16900000000000001,"15min":0.16},"max":{"1min":0.39200000000000002,"5min":0.42999999999999999,"15min":0.45600000000000002},"last":0.40500000000000003},{"interface":"front","average":{"1min":0.28399999999999997,"5min":0.28100000000000003,"15min":0.27200000000000002},"min":{"1min":0.17799999999999999,"5min":0.17799999999999999,"15min":0.17699999999999999},"max":{"1min":0.42599999999999999,"5min":0.47199999999999998,"15min":0.47199999999999998},"last":0.42399999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051834,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:47:01 2023","interfaces":[{"interface":"back","average":{"1min":0.27300000000000002,"5min":0.26100000000000001,"15min":0.27000000000000002},"min":{"1min":0.186,"5min":0.17399999999999999,"15min":0.17000000000000001},"max":{"1min":0.40400000000000003,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.25600000000000001},{"interface":"front","average":{"1min":0.28699999999999998,"5min":0.27500000000000002,"15min":0.28299999999999997},"min":{"1min":0.186,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.48599999999999999},"last":0.22800000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:47:49.510 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace:     ctx.managers[config['cluster']].wait_for_clean()
,   File "/home/teuthworker/src/github.com_ceph_ceph-c_511015c23eef0684a64171b710e432448df2b7c7/qa/tasks/ceph_manager.py", line 2747, in wait_for_clean
,     assert time.time() - start < timeout, \

Ignoring malparsed backtrace:  ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
,  1: /lib64/libpthread.so.0(+0x12cf0) [0x7f1453c2fcf0]
,  2: __pthread_mutex_lock()

Ignoring malparsed backtrace: 2023-03-25T04:28:05.172 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":142,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":5,"modified":"2023-03-25T04:27:12.092098+0000","services":{}},"progress_events":{}}
, 2023-03-25T04:28:05.185 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now
, 2023-03-25T04:28:05.186 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace: 2023-03-25T04:28:28.360 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":116,"stamp":"2023-03-25T04:28:28.108007+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002578"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.314661+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:27:45.314661+0000","last_peered":"2023-03-25T04:27:45.314661+0000","last_clean":"2023-03-25T04:27:45.314661+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:27:45.314661+0000","last_undegraded":"2023-03-25T04:27:45.314661+0000","last_fullsized":"2023-03-25T04:27:45.314661+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":25,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.310745+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:27:44.310745+0000","last_peered":"2023-03-25T04:27:44.310745+0000","last_clean":"2023-03-25T04:27:44.310745+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:27:44.310745+0000","last_undegraded":"2023-03-25T04:27:44.310745+0000","last_fullsized":"2023-03-25T04:27:44.310745+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":25,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.310650+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:27:44.310650+0000","last_peered":"2023-03-25T04:27:44.310650+0000","last_clean":"2023-03-25T04:27:44.310650+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:27:44.310650+0000","last_undegraded":"2023-03-25T04:27:44.310650+0000","last_fullsized":"2023-03-25T04:27:44.310650+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":26,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.312962+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:27:44.312962+0000","last_peered":"2023-03-25T04:27:44.312962+0000","last_clean":"2023-03-25T04:27:44.312962+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:27:44.312962+0000","last_undegraded":"2023-03-25T04:27:44.312962+0000","last_fullsized":"2023-03-25T04:27:44.312962+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315064+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:27:45.315064+0000","last_peered":"2023-03-25T04:27:45.315064+0000","last_clean":"2023-03-25T04:27:45.315064+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:27:45.315064+0000","last_undegraded":"2023-03-25T04:27:45.315064+0000","last_fullsized":"2023-03-25T04:27:45.315064+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":25,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.310707+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:27:44.310707+0000","last_peered":"2023-03-25T04:27:44.310707+0000","last_clean":"2023-03-25T04:27:44.310707+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:27:44.310707+0000","last_undegraded":"2023-03-25T04:27:44.310707+0000","last_fullsized":"2023-03-25T04:27:44.310707+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117007,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.375}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051597,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.186}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:28:28.362 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean
, 2023-03-25T04:28:28.362 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:28:45.655 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":124,"stamp":"2023-03-25T04:28:44.111265+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002492"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.314661+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:27:45.314661+0000","last_peered":"2023-03-25T04:27:45.314661+0000","last_clean":"2023-03-25T04:27:45.314661+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:27:45.314661+0000","last_undegraded":"2023-03-25T04:27:45.314661+0000","last_fullsized":"2023-03-25T04:27:45.314661+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":25,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.310745+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:27:44.310745+0000","last_peered":"2023-03-25T04:27:44.310745+0000","last_clean":"2023-03-25T04:27:44.310745+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:27:44.310745+0000","last_undegraded":"2023-03-25T04:27:44.310745+0000","last_fullsized":"2023-03-25T04:27:44.310745+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":25,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.310650+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:27:44.310650+0000","last_peered":"2023-03-25T04:27:44.310650+0000","last_clean":"2023-03-25T04:27:44.310650+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:27:44.310650+0000","last_undegraded":"2023-03-25T04:27:44.310650+0000","last_fullsized":"2023-03-25T04:27:44.310650+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":26,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.312962+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:27:44.312962+0000","last_peered":"2023-03-25T04:27:44.312962+0000","last_clean":"2023-03-25T04:27:44.312962+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:27:44.312962+0000","last_undegraded":"2023-03-25T04:27:44.312962+0000","last_fullsized":"2023-03-25T04:27:44.312962+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315064+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:27:45.315064+0000","last_peered":"2023-03-25T04:27:45.315064+0000","last_clean":"2023-03-25T04:27:45.315064+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:27:45.315064+0000","last_undegraded":"2023-03-25T04:27:45.315064+0000","last_fullsized":"2023-03-25T04:27:45.315064+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":25,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.310707+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:27:44.310707+0000","last_peered":"2023-03-25T04:27:44.310707+0000","last_clean":"2023-03-25T04:27:44.310707+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:27:44.310707+0000","last_undegraded":"2023-03-25T04:27:44.310707+0000","last_fullsized":"2023-03-25T04:27:44.310707+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117011,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:28:31 2023","interfaces":[{"interface":"back","average":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.36899999999999999},{"interface":"front","average":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"min":{"1min":0.19,"5min":0.19,"15min":0.19},"max":{"1min":0.46300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.39800000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051601,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:28:24 2023","interfaces":[{"interface":"back","average":{"1min":0.249,"5min":0.249,"15min":0.249},"min":{"1min":0.17699999999999999,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"last":0.254},{"interface":"front","average":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"min":{"1min":0.20200000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"last":0.312}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:28:45.656 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace: 2023-03-25T04:29:05.171 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}
, 2023-03-25T04:29:05.171 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json
, 2023-03-25T04:29:05.561 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:29:22.465 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:29:22.773 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all

Ignoring malparsed backtrace: 2023-03-25T04:29:42.405 INFO:teuthology.orchestra.run.smithi070.stdout:
, 2023-03-25T04:29:42.406 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":240,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":9,"modified":"2023-03-25T04:29:32.120358+0000","services":{"mon":{"daemons":{"summary":"","a":{"start_epoch":0,"start_stamp":"0.000000","gid":0,"addr":"(unrecognized address family 0)/0","metadata":{},"task_status":{}}}}}},"progress_events":{}}
, 2023-03-25T04:29:42.419 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:29:59.627 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":161,"stamp":"2023-03-25T04:29:58.125186+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002398"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315468+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:27:45.315468+0000","last_peered":"2023-03-25T04:27:45.315468+0000","last_clean":"2023-03-25T04:27:45.315468+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:27:45.315468+0000","last_undegraded":"2023-03-25T04:27:45.315468+0000","last_fullsized":"2023-03-25T04:27:45.315468+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312845+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:27:45.312845+0000","last_peered":"2023-03-25T04:27:45.312845+0000","last_clean":"2023-03-25T04:27:45.312845+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:27:45.312845+0000","last_undegraded":"2023-03-25T04:27:45.312845+0000","last_fullsized":"2023-03-25T04:27:45.312845+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312882+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:27:45.312882+0000","last_peered":"2023-03-25T04:27:45.312882+0000","last_clean":"2023-03-25T04:27:45.312882+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:27:45.312882+0000","last_undegraded":"2023-03-25T04:27:45.312882+0000","last_fullsized":"2023-03-25T04:27:45.312882+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.311730+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:27:45.311730+0000","last_peered":"2023-03-25T04:27:45.311730+0000","last_clean":"2023-03-25T04:27:45.311730+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:27:45.311730+0000","last_undegraded":"2023-03-25T04:27:45.311730+0000","last_fullsized":"2023-03-25T04:27:45.311730+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315735+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:27:45.315735+0000","last_peered":"2023-03-25T04:27:45.315735+0000","last_clean":"2023-03-25T04:27:45.315735+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:27:45.315735+0000","last_undegraded":"2023-03-25T04:27:45.315735+0000","last_fullsized":"2023-03-25T04:27:45.315735+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312833+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:27:45.312833+0000","last_peered":"2023-03-25T04:27:45.312833+0000","last_clean":"2023-03-25T04:27:45.312833+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:27:45.312833+0000","last_undegraded":"2023-03-25T04:27:45.312833+0000","last_fullsized":"2023-03-25T04:27:45.312833+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117026,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:29:33 2023","interfaces":[{"interface":"back","average":{"1min":0.26300000000000001,"5min":0.29099999999999998,"15min":0.29599999999999999},"min":{"1min":0.191,"5min":0.191,"15min":0.191},"max":{"1min":0.371,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.20499999999999999},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.29599999999999999,"15min":0.29999999999999999},"min":{"1min":0.21099999999999999,"5min":0.19,"15min":0.19},"max":{"1min":0.39800000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.19500000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051616,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:29:29 2023","interfaces":[{"interface":"back","average":{"1min":0.27200000000000002,"5min":0.253,"15min":0.25},"min":{"1min":0.183,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"last":0.26400000000000001},{"interface":"front","average":{"1min":0.28499999999999998,"5min":0.26800000000000002,"15min":0.26500000000000001},"min":{"1min":0.16600000000000001,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"last":0.29899999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:29:59.629 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean

Ignoring malparsed backtrace: 2023-03-25T04:30:16.889 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:30:16.903 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":170,"stamp":"2023-03-25T04:30:16.128427+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002222"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315468+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:27:45.315468+0000","last_peered":"2023-03-25T04:27:45.315468+0000","last_clean":"2023-03-25T04:27:45.315468+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:27:45.315468+0000","last_undegraded":"2023-03-25T04:27:45.315468+0000","last_fullsized":"2023-03-25T04:27:45.315468+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312845+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:27:45.312845+0000","last_peered":"2023-03-25T04:27:45.312845+0000","last_clean":"2023-03-25T04:27:45.312845+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:27:45.312845+0000","last_undegraded":"2023-03-25T04:27:45.312845+0000","last_fullsized":"2023-03-25T04:27:45.312845+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312882+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:27:45.312882+0000","last_peered":"2023-03-25T04:27:45.312882+0000","last_clean":"2023-03-25T04:27:45.312882+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:27:45.312882+0000","last_undegraded":"2023-03-25T04:27:45.312882+0000","last_fullsized":"2023-03-25T04:27:45.312882+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.311730+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:27:45.311730+0000","last_peered":"2023-03-25T04:27:45.311730+0000","last_clean":"2023-03-25T04:27:45.311730+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:27:45.311730+0000","last_undegraded":"2023-03-25T04:27:45.311730+0000","last_fullsized":"2023-03-25T04:27:45.311730+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315735+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:27:45.315735+0000","last_peered":"2023-03-25T04:27:45.315735+0000","last_clean":"2023-03-25T04:27:45.315735+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:27:45.315735+0000","last_undegraded":"2023-03-25T04:27:45.315735+0000","last_fullsized":"2023-03-25T04:27:45.315735+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312833+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:27:45.312833+0000","last_peered":"2023-03-25T04:27:45.312833+0000","last_clean":"2023-03-25T04:27:45.312833+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:27:45.312833+0000","last_undegraded":"2023-03-25T04:27:45.312833+0000","last_fullsized":"2023-03-25T04:27:45.312833+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117030,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:29:33 2023","interfaces":[{"interface":"back","average":{"1min":0.26300000000000001,"5min":0.29099999999999998,"15min":0.29599999999999999},"min":{"1min":0.191,"5min":0.191,"15min":0.191},"max":{"1min":0.371,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.245},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.29599999999999999,"15min":0.29999999999999999},"min":{"1min":0.21099999999999999,"5min":0.19,"15min":0.19},"max":{"1min":0.39800000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.20899999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051620,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:29:29 2023","interfaces":[{"interface":"back","average":{"1min":0.27200000000000002,"5min":0.253,"15min":0.25},"min":{"1min":0.183,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"last":0.19400000000000001},{"interface":"front","average":{"1min":0.28499999999999998,"5min":0.26800000000000002,"15min":0.26500000000000001},"min":{"1min":0.16600000000000001,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"last":0.23599999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:30:31.984+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:30:32.201+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:30:32.981+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)

Ignoring malparsed backtrace: 2023-03-25T04:30:48.449 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now
, 2023-03-25T04:30:48.450 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:30:48.760 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:31:08.342 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":196,"stamp":"2023-03-25T04:31:08.139018+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002431"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315468+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:27:45.315468+0000","last_peered":"2023-03-25T04:27:45.315468+0000","last_clean":"2023-03-25T04:27:45.315468+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:27:45.315468+0000","last_undegraded":"2023-03-25T04:27:45.315468+0000","last_fullsized":"2023-03-25T04:27:45.315468+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312845+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:27:45.312845+0000","last_peered":"2023-03-25T04:27:45.312845+0000","last_clean":"2023-03-25T04:27:45.312845+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:27:45.312845+0000","last_undegraded":"2023-03-25T04:27:45.312845+0000","last_fullsized":"2023-03-25T04:27:45.312845+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312882+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:27:45.312882+0000","last_peered":"2023-03-25T04:27:45.312882+0000","last_clean":"2023-03-25T04:27:45.312882+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:27:45.312882+0000","last_undegraded":"2023-03-25T04:27:45.312882+0000","last_fullsized":"2023-03-25T04:27:45.312882+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.311730+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:27:45.311730+0000","last_peered":"2023-03-25T04:27:45.311730+0000","last_clean":"2023-03-25T04:27:45.311730+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:27:45.311730+0000","last_undegraded":"2023-03-25T04:27:45.311730+0000","last_fullsized":"2023-03-25T04:27:45.311730+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315735+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:27:45.315735+0000","last_peered":"2023-03-25T04:27:45.315735+0000","last_clean":"2023-03-25T04:27:45.315735+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:27:45.315735+0000","last_undegraded":"2023-03-25T04:27:45.315735+0000","last_fullsized":"2023-03-25T04:27:45.315735+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312833+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:27:45.312833+0000","last_peered":"2023-03-25T04:27:45.312833+0000","last_clean":"2023-03-25T04:27:45.312833+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:27:45.312833+0000","last_undegraded":"2023-03-25T04:27:45.312833+0000","last_fullsized":"2023-03-25T04:27:45.312833+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117040,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:30:37 2023","interfaces":[{"interface":"back","average":{"1min":0.28799999999999998,"5min":0.28899999999999998,"15min":0.29499999999999998},"min":{"1min":0.20200000000000001,"5min":0.191,"15min":0.191},"max":{"1min":0.433,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.29799999999999999},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.28999999999999998,"15min":0.29799999999999999},"min":{"1min":0.17499999999999999,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":0.40300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.25700000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051630,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:30:32 2023","interfaces":[{"interface":"back","average":{"1min":0.25600000000000001,"5min":0.255,"15min":0.251},"min":{"1min":0.19400000000000001,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.35699999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"last":0.22800000000000001},{"interface":"front","average":{"1min":0.28699999999999998,"5min":0.27200000000000002,"15min":0.26600000000000001},"min":{"1min":0.20599999999999999,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.39400000000000002,"5min":0.42899999999999999,"15min":0.42899999999999999},"last":0.24099999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:31:08.343 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace: 2023-03-25T04:31:24.283+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:31:25.192+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:31:25.295+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)

Ignoring malparsed backtrace: 2023-03-25T04:31:40.257 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:31:40.271 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":212,"stamp":"2023-03-25T04:31:40.144732+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002208"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315468+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:27:45.315468+0000","last_peered":"2023-03-25T04:27:45.315468+0000","last_clean":"2023-03-25T04:27:45.315468+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:27:45.315468+0000","last_undegraded":"2023-03-25T04:27:45.315468+0000","last_fullsized":"2023-03-25T04:27:45.315468+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312845+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:27:45.312845+0000","last_peered":"2023-03-25T04:27:45.312845+0000","last_clean":"2023-03-25T04:27:45.312845+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:27:45.312845+0000","last_undegraded":"2023-03-25T04:27:45.312845+0000","last_fullsized":"2023-03-25T04:27:45.312845+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:23.308607+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:29:23.308607+0000","last_peered":"2023-03-25T04:29:23.308607+0000","last_clean":"2023-03-25T04:29:23.308607+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:29:23.308607+0000","last_undegraded":"2023-03-25T04:29:23.308607+0000","last_fullsized":"2023-03-25T04:29:23.308607+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":28,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:20.554710+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:29:20.554710+0000","last_peered":"2023-03-25T04:29:20.554710+0000","last_clean":"2023-03-25T04:29:20.554710+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:29:20.554710+0000","last_undegraded":"2023-03-25T04:29:20.554710+0000","last_fullsized":"2023-03-25T04:29:20.554710+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309216+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:29:28.309216+0000","last_peered":"2023-03-25T04:29:28.309216+0000","last_clean":"2023-03-25T04:29:28.309216+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:29:28.309216+0000","last_undegraded":"2023-03-25T04:29:28.309216+0000","last_fullsized":"2023-03-25T04:29:28.309216+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309233+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:29:28.309233+0000","last_peered":"2023-03-25T04:29:28.309233+0000","last_clean":"2023-03-25T04:29:28.309233+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:29:28.309233+0000","last_undegraded":"2023-03-25T04:29:28.309233+0000","last_fullsized":"2023-03-25T04:29:28.309233+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117046,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:30:37 2023","interfaces":[{"interface":"back","average":{"1min":0.28799999999999998,"5min":0.28899999999999998,"15min":0.29499999999999998},"min":{"1min":0.20200000000000001,"5min":0.191,"15min":0.191},"max":{"1min":0.433,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.216},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.28999999999999998,"15min":0.29799999999999999},"min":{"1min":0.17499999999999999,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":0.40300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.24299999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051637,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:31:35 2023","interfaces":[{"interface":"back","average":{"1min":0.26100000000000001,"5min":0.25700000000000001,"15min":0.251},"min":{"1min":0.19,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.35899999999999999,"5min":0.41599999999999998,"15min":0.41599999999999998},"last":0.221},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.27400000000000002,"15min":0.26700000000000002},"min":{"1min":0.20200000000000001,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.374,"5min":0.42899999999999999,"15min":0.42899999999999999},"last":0.24099999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:32:01.906+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:32:02.388+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:32:02.794 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:32:22.326 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":233,"stamp":"2023-03-25T04:32:22.153289+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002545"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:33.309748+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:29:33.309748+0000","last_peered":"2023-03-25T04:29:33.309748+0000","last_clean":"2023-03-25T04:29:33.309748+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:29:33.309748+0000","last_undegraded":"2023-03-25T04:29:33.309748+0000","last_fullsized":"2023-03-25T04:29:33.309748+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:33.309729+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:29:33.309729+0000","last_peered":"2023-03-25T04:29:33.309729+0000","last_clean":"2023-03-25T04:29:33.309729+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:29:33.309729+0000","last_undegraded":"2023-03-25T04:29:33.309729+0000","last_fullsized":"2023-03-25T04:29:33.309729+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:23.308607+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:29:23.308607+0000","last_peered":"2023-03-25T04:29:23.308607+0000","last_clean":"2023-03-25T04:29:23.308607+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:29:23.308607+0000","last_undegraded":"2023-03-25T04:29:23.308607+0000","last_fullsized":"2023-03-25T04:29:23.308607+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":28,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:20.554710+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:29:20.554710+0000","last_peered":"2023-03-25T04:29:20.554710+0000","last_clean":"2023-03-25T04:29:20.554710+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:29:20.554710+0000","last_undegraded":"2023-03-25T04:29:20.554710+0000","last_fullsized":"2023-03-25T04:29:20.554710+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309216+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:29:28.309216+0000","last_peered":"2023-03-25T04:29:28.309216+0000","last_clean":"2023-03-25T04:29:28.309216+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:29:28.309216+0000","last_undegraded":"2023-03-25T04:29:28.309216+0000","last_fullsized":"2023-03-25T04:29:28.309216+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309233+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:29:28.309233+0000","last_peered":"2023-03-25T04:29:28.309233+0000","last_clean":"2023-03-25T04:29:28.309233+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:29:28.309233+0000","last_undegraded":"2023-03-25T04:29:28.309233+0000","last_fullsized":"2023-03-25T04:29:28.309233+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117055,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:31:42 2023","interfaces":[{"interface":"back","average":{"1min":0.27500000000000002,"5min":0.28399999999999997,"15min":0.29399999999999998},"min":{"1min":0.19800000000000001,"5min":0.191,"15min":0.191},"max":{"1min":0.36899999999999999,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.32300000000000001},{"interface":"front","average":{"1min":0.27900000000000003,"5min":0.28499999999999998,"15min":0.29599999999999999},"min":{"1min":0.216,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":0.40799999999999997,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.27100000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051646,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:31:35 2023","interfaces":[{"interface":"back","average":{"1min":0.26100000000000001,"5min":0.25700000000000001,"15min":0.251},"min":{"1min":0.19,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.35899999999999999,"5min":0.41599999999999998,"15min":0.41599999999999998},"last":0.25600000000000001},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.27400000000000002,"15min":0.26700000000000002},"min":{"1min":0.20200000000000001,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.374,"5min":0.42899999999999999,"15min":0.42899999999999999},"last":0.29199999999999998}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:32:22.327 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean
, 2023-03-25T04:32:22.327 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}

Ignoring malparsed backtrace: 2023-03-25T04:32:39.836 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:32:39.849 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":241,"stamp":"2023-03-25T04:32:38.156312+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002346"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:33.309748+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:29:33.309748+0000","last_peered":"2023-03-25T04:29:33.309748+0000","last_clean":"2023-03-25T04:29:33.309748+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:29:33.309748+0000","last_undegraded":"2023-03-25T04:29:33.309748+0000","last_fullsized":"2023-03-25T04:29:33.309748+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:33.309729+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:29:33.309729+0000","last_peered":"2023-03-25T04:29:33.309729+0000","last_clean":"2023-03-25T04:29:33.309729+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:29:33.309729+0000","last_undegraded":"2023-03-25T04:29:33.309729+0000","last_fullsized":"2023-03-25T04:29:33.309729+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:23.308607+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:29:23.308607+0000","last_peered":"2023-03-25T04:29:23.308607+0000","last_clean":"2023-03-25T04:29:23.308607+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:29:23.308607+0000","last_undegraded":"2023-03-25T04:29:23.308607+0000","last_fullsized":"2023-03-25T04:29:23.308607+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":28,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:20.554710+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:29:20.554710+0000","last_peered":"2023-03-25T04:29:20.554710+0000","last_clean":"2023-03-25T04:29:20.554710+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:29:20.554710+0000","last_undegraded":"2023-03-25T04:29:20.554710+0000","last_fullsized":"2023-03-25T04:29:20.554710+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309216+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:29:28.309216+0000","last_peered":"2023-03-25T04:29:28.309216+0000","last_clean":"2023-03-25T04:29:28.309216+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:29:28.309216+0000","last_undegraded":"2023-03-25T04:29:28.309216+0000","last_fullsized":"2023-03-25T04:29:28.309216+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309233+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:29:28.309233+0000","last_peered":"2023-03-25T04:29:28.309233+0000","last_clean":"2023-03-25T04:29:28.309233+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:29:28.309233+0000","last_undegraded":"2023-03-25T04:29:28.309233+0000","last_fullsized":"2023-03-25T04:29:28.309233+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117058,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:31:42 2023","interfaces":[{"interface":"back","average":{"1min":0.27500000000000002,"5min":0.28399999999999997,"15min":0.29399999999999998},"min":{"1min":0.19800000000000001,"5min":0.191,"15min":0.191},"max":{"1min":0.36899999999999999,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.30599999999999999},{"interface":"front","average":{"1min":0.27900000000000003,"5min":0.28499999999999998,"15min":0.29599999999999999},"min":{"1min":0.216,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":0.40799999999999997,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.32500000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051649,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:31:35 2023","interfaces":[{"interface":"back","average":{"1min":0.26100000000000001,"5min":0.25700000000000001,"15min":0.251},"min":{"1min":0.19,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.35899999999999999,"5min":0.41599999999999998,"15min":0.41599999999999998},"last":0.24199999999999999},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.27400000000000002,"15min":0.26700000000000002},"min":{"1min":0.20200000000000001,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.374,"5min":0.42899999999999999,"15min":0.42899999999999999},"last":0.22600000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:32:54.944+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:32:55.469+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:32:55.927+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)

Ignoring malparsed backtrace: 2023-03-25T04:33:13.958 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean
, 2023-03-25T04:33:13.959 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}
, 2023-03-25T04:33:13.959 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json

Ignoring malparsed backtrace: 2023-03-25T04:33:31.199 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":267,"stamp":"2023-03-25T04:33:30.166660+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002269"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:33.309748+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:29:33.309748+0000","last_peered":"2023-03-25T04:29:33.309748+0000","last_clean":"2023-03-25T04:29:33.309748+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:29:33.309748+0000","last_undegraded":"2023-03-25T04:29:33.309748+0000","last_fullsized":"2023-03-25T04:29:33.309748+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:33.309729+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:29:33.309729+0000","last_peered":"2023-03-25T04:29:33.309729+0000","last_clean":"2023-03-25T04:29:33.309729+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:29:33.309729+0000","last_undegraded":"2023-03-25T04:29:33.309729+0000","last_fullsized":"2023-03-25T04:29:33.309729+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":28,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:31:23.320894+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:31:23.320894+0000","last_peered":"2023-03-25T04:31:23.320894+0000","last_clean":"2023-03-25T04:31:23.320894+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:31:23.320894+0000","last_undegraded":"2023-03-25T04:31:23.320894+0000","last_fullsized":"2023-03-25T04:31:23.320894+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":28,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:20.554710+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:29:20.554710+0000","last_peered":"2023-03-25T04:29:20.554710+0000","last_clean":"2023-03-25T04:29:20.554710+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:29:20.554710+0000","last_undegraded":"2023-03-25T04:29:20.554710+0000","last_fullsized":"2023-03-25T04:29:20.554710+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309216+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:29:28.309216+0000","last_peered":"2023-03-25T04:29:28.309216+0000","last_clean":"2023-03-25T04:29:28.309216+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:29:28.309216+0000","last_undegraded":"2023-03-25T04:29:28.309216+0000","last_fullsized":"2023-03-25T04:29:28.309216+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309233+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:29:28.309233+0000","last_peered":"2023-03-25T04:29:28.309233+0000","last_clean":"2023-03-25T04:29:28.309233+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:29:28.309233+0000","last_undegraded":"2023-03-25T04:29:28.309233+0000","last_fullsized":"2023-03-25T04:29:28.309233+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117068,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:32:46 2023","interfaces":[{"interface":"back","average":{"1min":0.27000000000000002,"5min":0.27900000000000003,"15min":0.29199999999999998},"min":{"1min":0.17899999999999999,"5min":0.17899999999999999,"15min":0.17899999999999999},"max":{"1min":0.35599999999999998,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.28799999999999998},{"interface":"front","average":{"1min":0.26800000000000002,"5min":0.27800000000000002,"15min":0.29399999999999998},"min":{"1min":0.19900000000000001,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":0.377,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.315}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051660,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:32:36 2023","interfaces":[{"interface":"back","average":{"1min":0.248,"5min":0.25700000000000001,"15min":0.251},"min":{"1min":0.17000000000000001,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"last":0.22700000000000001},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.27600000000000002,"15min":0.26800000000000002},"min":{"1min":0.192,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"last":0.20399999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:33:31.201 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:33:31.400+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)

Ignoring malparsed backtrace: 2023-03-25T04:33:47.531+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:33:47.951+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:33:48.505+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)

Ignoring malparsed backtrace: 2023-03-25T04:34:06.104 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json
, 2023-03-25T04:34:06.118+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:34:06.497 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:34:23.399 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:34:23.665+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:34:23.712 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:34:40.195+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:34:40.523+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:34:40.676 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace: 2023-03-25T04:35:03.381+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:35:04.139+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:35:04.391+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)

Ignoring malparsed backtrace: 2023-03-25T04:35:22.738 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean
, 2023-03-25T04:35:22.739 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}
, 2023-03-25T04:35:22.739 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json

Ignoring malparsed backtrace: 2023-03-25T04:35:41.393+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:35:42.352+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:35:42.376+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:35:57.384 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":340,"stamp":"2023-03-25T04:35:56.194992+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.001930"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":36,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:31:43.322894+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:31:43.322894+0000","last_peered":"2023-03-25T04:31:43.322894+0000","last_clean":"2023-03-25T04:31:43.322894+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:31:43.322894+0000","last_undegraded":"2023-03-25T04:31:43.322894+0000","last_fullsized":"2023-03-25T04:31:43.322894+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":28,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:31:43.322876+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:31:43.322876+0000","last_peered":"2023-03-25T04:31:43.322876+0000","last_clean":"2023-03-25T04:31:43.322876+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:31:43.322876+0000","last_undegraded":"2023-03-25T04:31:43.322876+0000","last_fullsized":"2023-03-25T04:31:43.322876+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:23.331998+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:33:23.331998+0000","last_peered":"2023-03-25T04:33:23.331998+0000","last_clean":"2023-03-25T04:33:23.331998+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:33:23.331998+0000","last_undegraded":"2023-03-25T04:33:23.331998+0000","last_fullsized":"2023-03-25T04:33:23.331998+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:30.576723+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:33:30.576723+0000","last_peered":"2023-03-25T04:33:30.576723+0000","last_clean":"2023-03-25T04:33:30.576723+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:33:30.576723+0000","last_undegraded":"2023-03-25T04:33:30.576723+0000","last_fullsized":"2023-03-25T04:33:30.576723+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":37,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:38.333447+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:33:38.333447+0000","last_peered":"2023-03-25T04:33:38.333447+0000","last_clean":"2023-03-25T04:33:38.333447+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:33:38.333447+0000","last_undegraded":"2023-03-25T04:33:38.333447+0000","last_fullsized":"2023-03-25T04:33:38.333447+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:38.333467+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:33:38.333467+0000","last_peered":"2023-03-25T04:33:38.333467+0000","last_clean":"2023-03-25T04:33:38.333467+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:33:38.333467+0000","last_undegraded":"2023-03-25T04:33:38.333467+0000","last_fullsized":"2023-03-25T04:33:38.333467+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117098,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:35:49 2023","interfaces":[{"interface":"back","average":{"1min":0.255,"5min":0.26000000000000001,"15min":0.28199999999999997},"min":{"1min":0.17899999999999999,"5min":0.16,"15min":0.16},"max":{"1min":0.36399999999999999,"5min":0.42599999999999999,"15min":0.45100000000000001},"last":0.26100000000000001},{"interface":"front","average":{"1min":0.26000000000000001,"5min":0.26700000000000002,"15min":0.28599999999999998},"min":{"1min":0.188,"5min":0.17999999999999999,"15min":0.17499999999999999},"max":{"1min":0.39100000000000001,"5min":0.45200000000000001,"15min":0.46300000000000002},"last":0.28399999999999997}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051689,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:35:42 2023","interfaces":[{"interface":"back","average":{"1min":0.29399999999999998,"5min":0.27200000000000002,"15min":0.25800000000000001},"min":{"1min":0.17299999999999999,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.41099999999999998,"5min":0.42699999999999999,"15min":0.42699999999999999},"last":0.20399999999999999},{"interface":"front","average":{"1min":0.30299999999999999,"5min":0.28399999999999997,"15min":0.27300000000000002},"min":{"1min":0.19700000000000001,"5min":0.192,"15min":0.16600000000000001},"max":{"1min":0.43099999999999999,"5min":0.45800000000000002,"15min":0.45800000000000002},"last":0.22}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:35:57.624+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)

Ignoring malparsed backtrace: 2023-03-25T04:36:17.226 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:36:17.239 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":350,"stamp":"2023-03-25T04:36:16.198866+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002148"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":37,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:53.335026+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:33:53.335026+0000","last_peered":"2023-03-25T04:33:53.335026+0000","last_clean":"2023-03-25T04:33:53.335026+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:33:53.335026+0000","last_undegraded":"2023-03-25T04:33:53.335026+0000","last_fullsized":"2023-03-25T04:33:53.335026+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:53.335009+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:33:53.335009+0000","last_peered":"2023-03-25T04:33:53.335009+0000","last_clean":"2023-03-25T04:33:53.335009+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:33:53.335009+0000","last_undegraded":"2023-03-25T04:33:53.335009+0000","last_fullsized":"2023-03-25T04:33:53.335009+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:23.331998+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:33:23.331998+0000","last_peered":"2023-03-25T04:33:23.331998+0000","last_clean":"2023-03-25T04:33:23.331998+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:33:23.331998+0000","last_undegraded":"2023-03-25T04:33:23.331998+0000","last_fullsized":"2023-03-25T04:33:23.331998+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:30.576723+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:33:30.576723+0000","last_peered":"2023-03-25T04:33:30.576723+0000","last_clean":"2023-03-25T04:33:30.576723+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:33:30.576723+0000","last_undegraded":"2023-03-25T04:33:30.576723+0000","last_fullsized":"2023-03-25T04:33:30.576723+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":37,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:38.333447+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:33:38.333447+0000","last_peered":"2023-03-25T04:33:38.333447+0000","last_clean":"2023-03-25T04:33:38.333447+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:33:38.333447+0000","last_undegraded":"2023-03-25T04:33:38.333447+0000","last_fullsized":"2023-03-25T04:33:38.333447+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:38.333467+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:33:38.333467+0000","last_peered":"2023-03-25T04:33:38.333467+0000","last_clean":"2023-03-25T04:33:38.333467+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:33:38.333467+0000","last_undegraded":"2023-03-25T04:33:38.333467+0000","last_fullsized":"2023-03-25T04:33:38.333467+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117102,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:35:49 2023","interfaces":[{"interface":"back","average":{"1min":0.255,"5min":0.26000000000000001,"15min":0.28199999999999997},"min":{"1min":0.17899999999999999,"5min":0.16,"15min":0.16},"max":{"1min":0.36399999999999999,"5min":0.42599999999999999,"15min":0.45100000000000001},"last":0.28399999999999997},{"interface":"front","average":{"1min":0.26000000000000001,"5min":0.26700000000000002,"15min":0.28599999999999998},"min":{"1min":0.188,"5min":0.17999999999999999,"15min":0.17499999999999999},"max":{"1min":0.39100000000000001,"5min":0.45200000000000001,"15min":0.46300000000000002},"last":0.30299999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051693,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:35:42 2023","interfaces":[{"interface":"back","average":{"1min":0.29399999999999998,"5min":0.27200000000000002,"15min":0.25800000000000001},"min":{"1min":0.17299999999999999,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.41099999999999998,"5min":0.42699999999999999,"15min":0.42699999999999999},"last":0.35099999999999998},{"interface":"front","average":{"1min":0.30299999999999999,"5min":0.28399999999999997,"15min":0.27300000000000002},"min":{"1min":0.19700000000000001,"5min":0.192,"15min":0.16600000000000001},"max":{"1min":0.43099999999999999,"5min":0.45800000000000002,"15min":0.45800000000000002},"last":0.32900000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:36:34.556+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:36:34.717+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:36:35.583+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)

Ignoring malparsed backtrace: 2023-03-25T04:36:52.680 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json
, 2023-03-25T04:36:52.701+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:36:53.067 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:37:09.963 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:37:10.273 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all

Ignoring malparsed backtrace: 2023-03-25T04:37:26.856+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:37:27.288 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:37:27.538+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:37:41.903 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":392,"stamp":"2023-03-25T04:37:40.213373+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002012"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":37,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:53.335026+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:33:53.335026+0000","last_peered":"2023-03-25T04:33:53.335026+0000","last_clean":"2023-03-25T04:33:53.335026+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:33:53.335026+0000","last_undegraded":"2023-03-25T04:33:53.335026+0000","last_fullsized":"2023-03-25T04:33:53.335026+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:53.335009+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:33:53.335009+0000","last_peered":"2023-03-25T04:33:53.335009+0000","last_clean":"2023-03-25T04:33:53.335009+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:33:53.335009+0000","last_undegraded":"2023-03-25T04:33:53.335009+0000","last_fullsized":"2023-03-25T04:33:53.335009+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:23.343585+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:35:23.343585+0000","last_peered":"2023-03-25T04:35:23.343585+0000","last_clean":"2023-03-25T04:35:23.343585+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:35:23.343585+0000","last_undegraded":"2023-03-25T04:35:23.343585+0000","last_fullsized":"2023-03-25T04:35:23.343585+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:30.576723+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:33:30.576723+0000","last_peered":"2023-03-25T04:33:30.576723+0000","last_clean":"2023-03-25T04:33:30.576723+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:33:30.576723+0000","last_undegraded":"2023-03-25T04:33:30.576723+0000","last_fullsized":"2023-03-25T04:33:30.576723+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":37,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:38.333447+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:33:38.333447+0000","last_peered":"2023-03-25T04:33:38.333447+0000","last_clean":"2023-03-25T04:33:38.333447+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:33:38.333447+0000","last_undegraded":"2023-03-25T04:33:38.333447+0000","last_fullsized":"2023-03-25T04:33:38.333447+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:38.333467+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:33:38.333467+0000","last_peered":"2023-03-25T04:33:38.333467+0000","last_clean":"2023-03-25T04:33:38.333467+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:33:38.333467+0000","last_undegraded":"2023-03-25T04:33:38.333467+0000","last_fullsized":"2023-03-25T04:33:38.333467+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117119,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:36:50 2023","interfaces":[{"interface":"back","average":{"1min":0.254,"5min":0.25600000000000001,"15min":0.27900000000000003},"min":{"1min":0.20499999999999999,"5min":0.16,"15min":0.16},"max":{"1min":0.371,"5min":0.42599999999999999,"15min":0.45100000000000001},"last":0.23000000000000001},{"interface":"front","average":{"1min":0.26800000000000002,"5min":0.26500000000000001,"15min":0.28399999999999997},"min":{"1min":0.19700000000000001,"5min":0.17999999999999999,"15min":0.17499999999999999},"max":{"1min":0.34699999999999998,"5min":0.45200000000000001,"15min":0.46300000000000002},"last":0.20899999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051710,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:36:44 2023","interfaces":[{"interface":"back","average":{"1min":0.28399999999999997,"5min":0.27700000000000002,"15min":0.26100000000000001},"min":{"1min":0.192,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.40899999999999997,"5min":0.42699999999999999,"15min":0.42699999999999999},"last":0.39700000000000002},{"interface":"front","average":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.27500000000000002},"min":{"1min":0.182,"5min":0.182,"15min":0.16600000000000001},"max":{"1min":0.439,"5min":0.45800000000000002,"15min":0.45800000000000002},"last":0.439}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:37:42.400+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)

Ignoring malparsed backtrace: 2023-03-25T04:38:04.766+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:38:04.922 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:38:05.246 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:38:19.605 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":411,"stamp":"2023-03-25T04:38:18.220966+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002368"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":38,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:36:03.347070+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:36:03.347070+0000","last_peered":"2023-03-25T04:36:03.347070+0000","last_clean":"2023-03-25T04:36:03.347070+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:36:03.347070+0000","last_undegraded":"2023-03-25T04:36:03.347070+0000","last_fullsized":"2023-03-25T04:36:03.347070+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:36:03.347025+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:36:03.347025+0000","last_peered":"2023-03-25T04:36:03.347025+0000","last_clean":"2023-03-25T04:36:03.347025+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:36:03.347025+0000","last_undegraded":"2023-03-25T04:36:03.347025+0000","last_fullsized":"2023-03-25T04:36:03.347025+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:23.343585+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:35:23.343585+0000","last_peered":"2023-03-25T04:35:23.343585+0000","last_clean":"2023-03-25T04:35:23.343585+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:35:23.343585+0000","last_undegraded":"2023-03-25T04:35:23.343585+0000","last_fullsized":"2023-03-25T04:35:23.343585+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:35.587025+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:35:35.587025+0000","last_peered":"2023-03-25T04:35:35.587025+0000","last_clean":"2023-03-25T04:35:35.587025+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:35:35.587025+0000","last_undegraded":"2023-03-25T04:35:35.587025+0000","last_fullsized":"2023-03-25T04:35:35.587025+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":38,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:43.345308+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:35:43.345308+0000","last_peered":"2023-03-25T04:35:43.345308+0000","last_clean":"2023-03-25T04:35:43.345308+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:35:43.345308+0000","last_undegraded":"2023-03-25T04:35:43.345308+0000","last_fullsized":"2023-03-25T04:35:43.345308+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:43.345324+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:35:43.345324+0000","last_peered":"2023-03-25T04:35:43.345324+0000","last_clean":"2023-03-25T04:35:43.345324+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:35:43.345324+0000","last_undegraded":"2023-03-25T04:35:43.345324+0000","last_fullsized":"2023-03-25T04:35:43.345324+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117127,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:37:53 2023","interfaces":[{"interface":"back","average":{"1min":0.25800000000000001,"5min":0.253,"15min":0.27700000000000002},"min":{"1min":0.184,"5min":0.16,"15min":0.16},"max":{"1min":0.44600000000000001,"5min":0.44600000000000001,"15min":0.45100000000000001},"last":0.35199999999999998},{"interface":"front","average":{"1min":0.26900000000000002,"5min":0.26500000000000001,"15min":0.28199999999999997},"min":{"1min":0.192,"5min":0.17999999999999999,"15min":0.17499999999999999},"max":{"1min":0.46400000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"last":0.33200000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051717,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:37:44 2023","interfaces":[{"interface":"back","average":{"1min":0.27300000000000002,"5min":0.28199999999999997,"15min":0.26200000000000001},"min":{"1min":0.17899999999999999,"5min":0.17299999999999999,"15min":0.17000000000000001},"max":{"1min":0.41799999999999998,"5min":0.42099999999999999,"15min":0.42699999999999999},"last":0.28699999999999998},{"interface":"front","average":{"1min":0.29399999999999998,"5min":0.29099999999999998,"15min":0.27700000000000002},"min":{"1min":0.19900000000000001,"5min":0.182,"15min":0.16600000000000001},"max":{"1min":0.441,"5min":0.441,"15min":0.45800000000000002},"last":0.26300000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:38:19.718+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:38:20.315+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)

Ignoring malparsed backtrace: 2023-03-25T04:38:44.283 INFO:teuthology.orchestra.run.smithi070.stdout:
, 2023-03-25T04:38:44.283 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":781,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":16,"modified":"2023-03-25T04:38:40.225321+0000","services":{}},"progress_events":{}}
, 2023-03-25T04:38:44.296 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now

Ignoring malparsed backtrace: 2023-03-25T04:39:02.477+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:39:02.578+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:39:03.524+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:39:20.359 INFO:teuthology.orchestra.run.smithi070.stdout:
, 2023-03-25T04:39:20.359 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":818,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":16,"modified":"2023-03-25T04:38:40.225321+0000","services":{}},"progress_events":{}}
, 2023-03-25T04:39:20.372 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now

Ignoring malparsed backtrace: 2023-03-25T04:39:37.527 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:39:37.541 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":450,"stamp":"2023-03-25T04:39:36.235332+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002086"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":38,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:36:03.347070+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:36:03.347070+0000","last_peered":"2023-03-25T04:36:03.347070+0000","last_clean":"2023-03-25T04:36:03.347070+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:36:03.347070+0000","last_undegraded":"2023-03-25T04:36:03.347070+0000","last_fullsized":"2023-03-25T04:36:03.347070+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:36:03.347025+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:36:03.347025+0000","last_peered":"2023-03-25T04:36:03.347025+0000","last_clean":"2023-03-25T04:36:03.347025+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:36:03.347025+0000","last_undegraded":"2023-03-25T04:36:03.347025+0000","last_fullsized":"2023-03-25T04:36:03.347025+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:23.353778+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:37:23.353778+0000","last_peered":"2023-03-25T04:37:23.353778+0000","last_clean":"2023-03-25T04:37:23.353778+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:37:23.353778+0000","last_undegraded":"2023-03-25T04:37:23.353778+0000","last_fullsized":"2023-03-25T04:37:23.353778+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:35.587025+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:35:35.587025+0000","last_peered":"2023-03-25T04:35:35.587025+0000","last_clean":"2023-03-25T04:35:35.587025+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:35:35.587025+0000","last_undegraded":"2023-03-25T04:35:35.587025+0000","last_fullsized":"2023-03-25T04:35:35.587025+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":38,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:43.345308+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:35:43.345308+0000","last_peered":"2023-03-25T04:35:43.345308+0000","last_clean":"2023-03-25T04:35:43.345308+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:35:43.345308+0000","last_undegraded":"2023-03-25T04:35:43.345308+0000","last_fullsized":"2023-03-25T04:35:43.345308+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:43.345324+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:35:43.345324+0000","last_peered":"2023-03-25T04:35:43.345324+0000","last_clean":"2023-03-25T04:35:43.345324+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:35:43.345324+0000","last_undegraded":"2023-03-25T04:35:43.345324+0000","last_fullsized":"2023-03-25T04:35:43.345324+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117143,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:38:54 2023","interfaces":[{"interface":"back","average":{"1min":0.25800000000000001,"5min":0.25700000000000001,"15min":0.27400000000000002},"min":{"1min":0.192,"5min":0.17399999999999999,"15min":0.16},"max":{"1min":0.39400000000000002,"5min":0.44600000000000001,"15min":0.45100000000000001},"last":0.26500000000000001},{"interface":"front","average":{"1min":0.25900000000000001,"5min":0.26700000000000002,"15min":0.27900000000000003},"min":{"1min":0.19500000000000001,"5min":0.188,"15min":0.17499999999999999},"max":{"1min":0.41499999999999998,"5min":0.46400000000000002,"15min":0.46400000000000002},"last":0.29099999999999998}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051733,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:38:45 2023","interfaces":[{"interface":"back","average":{"1min":0.27300000000000002,"5min":0.28100000000000003,"15min":0.26400000000000001},"min":{"1min":0.19400000000000001,"5min":0.17299999999999999,"15min":0.17000000000000001},"max":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"last":0.27300000000000002},{"interface":"front","average":{"1min":0.28799999999999998,"5min":0.29099999999999998,"15min":0.27800000000000002},"min":{"1min":0.19800000000000001,"5min":0.182,"15min":0.16600000000000001},"max":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"last":0.308}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:39:54.416+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:39:54.455 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:39:54.780 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:40:09.288+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:40:09.389+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:40:10.267+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)

Ignoring malparsed backtrace: 2023-03-25T04:40:28.984 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":476,"stamp":"2023-03-25T04:40:28.245911+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002693"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":39,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:38:13.358968+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:38:13.358968+0000","last_peered":"2023-03-25T04:38:13.358968+0000","last_clean":"2023-03-25T04:38:13.358968+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:38:13.358968+0000","last_undegraded":"2023-03-25T04:38:13.358968+0000","last_fullsized":"2023-03-25T04:38:13.358968+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:38:13.358922+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:38:13.358922+0000","last_peered":"2023-03-25T04:38:13.358922+0000","last_clean":"2023-03-25T04:38:13.358922+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:38:13.358922+0000","last_undegraded":"2023-03-25T04:38:13.358922+0000","last_fullsized":"2023-03-25T04:38:13.358922+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:23.353778+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:37:23.353778+0000","last_peered":"2023-03-25T04:37:23.353778+0000","last_clean":"2023-03-25T04:37:23.353778+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:37:23.353778+0000","last_undegraded":"2023-03-25T04:37:23.353778+0000","last_fullsized":"2023-03-25T04:37:23.353778+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:40.598579+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:37:40.598579+0000","last_peered":"2023-03-25T04:37:40.598579+0000","last_clean":"2023-03-25T04:37:40.598579+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:37:40.598579+0000","last_undegraded":"2023-03-25T04:37:40.598579+0000","last_fullsized":"2023-03-25T04:37:40.598579+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":39,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:48.356344+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:37:48.356344+0000","last_peered":"2023-03-25T04:37:48.356344+0000","last_clean":"2023-03-25T04:37:48.356344+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:37:48.356344+0000","last_undegraded":"2023-03-25T04:37:48.356344+0000","last_fullsized":"2023-03-25T04:37:48.356344+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:48.356365+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:37:48.356365+0000","last_peered":"2023-03-25T04:37:48.356365+0000","last_clean":"2023-03-25T04:37:48.356365+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:37:48.356365+0000","last_undegraded":"2023-03-25T04:37:48.356365+0000","last_fullsized":"2023-03-25T04:37:48.356365+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117153,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:39:54 2023","interfaces":[{"interface":"back","average":{"1min":0.25900000000000001,"5min":0.25600000000000001,"15min":0.27100000000000002},"min":{"1min":0.188,"5min":0.17899999999999999,"15min":0.16},"max":{"1min":0.41999999999999998,"5min":0.44600000000000001,"15min":0.45100000000000001},"last":0.255},{"interface":"front","average":{"1min":0.27500000000000002,"5min":0.26600000000000001,"15min":0.27700000000000002},"min":{"1min":0.17699999999999999,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.45600000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"last":0.26800000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051743,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:39:47 2023","interfaces":[{"interface":"back","average":{"1min":0.26300000000000001,"5min":0.27700000000000002,"15min":0.26500000000000001},"min":{"1min":0.187,"5min":0.17299999999999999,"15min":0.17000000000000001},"max":{"1min":0.433,"5min":0.433,"15min":0.433},"last":0.36599999999999999},{"interface":"front","average":{"1min":0.27100000000000002,"5min":0.28799999999999998,"15min":0.27900000000000003},"min":{"1min":0.20100000000000001,"5min":0.182,"15min":0.16600000000000001},"max":{"1min":0.48599999999999999,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.34999999999999998}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:40:28.986 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean
, 2023-03-25T04:40:28.986 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}

Ignoring malparsed backtrace: 2023-03-25T04:40:47.229+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:40:47.503+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:40:48.251+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)

Ignoring malparsed backtrace: 2023-03-25T04:41:03.267 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:41:03.578 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all

Ignoring malparsed backtrace: 2023-03-25T04:41:21.347 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":502,"stamp":"2023-03-25T04:41:20.256204+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002342"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":39,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:38:13.358968+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:38:13.358968+0000","last_peered":"2023-03-25T04:38:13.358968+0000","last_clean":"2023-03-25T04:38:13.358968+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:38:13.358968+0000","last_undegraded":"2023-03-25T04:38:13.358968+0000","last_fullsized":"2023-03-25T04:38:13.358968+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:38:13.358922+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:38:13.358922+0000","last_peered":"2023-03-25T04:38:13.358922+0000","last_clean":"2023-03-25T04:38:13.358922+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:38:13.358922+0000","last_undegraded":"2023-03-25T04:38:13.358922+0000","last_fullsized":"2023-03-25T04:38:13.358922+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:23.353778+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:37:23.353778+0000","last_peered":"2023-03-25T04:37:23.353778+0000","last_clean":"2023-03-25T04:37:23.353778+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:37:23.353778+0000","last_undegraded":"2023-03-25T04:37:23.353778+0000","last_fullsized":"2023-03-25T04:37:23.353778+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:40.598579+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:37:40.598579+0000","last_peered":"2023-03-25T04:37:40.598579+0000","last_clean":"2023-03-25T04:37:40.598579+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:37:40.598579+0000","last_undegraded":"2023-03-25T04:37:40.598579+0000","last_fullsized":"2023-03-25T04:37:40.598579+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":39,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:48.356344+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:37:48.356344+0000","last_peered":"2023-03-25T04:37:48.356344+0000","last_clean":"2023-03-25T04:37:48.356344+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:37:48.356344+0000","last_undegraded":"2023-03-25T04:37:48.356344+0000","last_fullsized":"2023-03-25T04:37:48.356344+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:48.356365+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:37:48.356365+0000","last_peered":"2023-03-25T04:37:48.356365+0000","last_clean":"2023-03-25T04:37:48.356365+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:37:48.356365+0000","last_undegraded":"2023-03-25T04:37:48.356365+0000","last_fullsized":"2023-03-25T04:37:48.356365+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117163,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:40:56 2023","interfaces":[{"interface":"back","average":{"1min":0.28799999999999998,"5min":0.26300000000000001,"15min":0.27100000000000002},"min":{"1min":0.188,"5min":0.184,"15min":0.16},"max":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.17499999999999999},{"interface":"front","average":{"1min":0.29599999999999999,"5min":0.27300000000000002,"15min":0.27700000000000002},"min":{"1min":0.20599999999999999,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.43099999999999999,"5min":0.46400000000000002,"15min":0.46400000000000002},"last":0.20699999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051754,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:40:52 2023","interfaces":[{"interface":"back","average":{"1min":0.28599999999999998,"5min":0.27500000000000002,"15min":0.26700000000000002},"min":{"1min":0.17899999999999999,"5min":0.17899999999999999,"15min":0.17000000000000001},"max":{"1min":0.44,"5min":0.44,"15min":0.44},"last":0.23699999999999999},{"interface":"front","average":{"1min":0.29799999999999999,"5min":0.28699999999999998,"15min":0.28100000000000003},"min":{"1min":0.20300000000000001,"5min":0.182,"15min":0.16600000000000001},"max":{"1min":0.47599999999999998,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.28000000000000003}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:41:21.349 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean
, 2023-03-25T04:41:21.349 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}

Ignoring malparsed backtrace: 2023-03-25T04:41:39.867+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:41:40.383+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:41:40.905+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:41:58.219 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json
, 2023-03-25T04:41:58.319+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:41:58.644 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:42:18.193 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":530,"stamp":"2023-03-25T04:42:16.266659+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002356"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":39,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:38:13.358968+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:38:13.358968+0000","last_peered":"2023-03-25T04:38:13.358968+0000","last_clean":"2023-03-25T04:38:13.358968+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:38:13.358968+0000","last_undegraded":"2023-03-25T04:38:13.358968+0000","last_fullsized":"2023-03-25T04:38:13.358968+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:38:13.358922+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:38:13.358922+0000","last_peered":"2023-03-25T04:38:13.358922+0000","last_clean":"2023-03-25T04:38:13.358922+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:38:13.358922+0000","last_undegraded":"2023-03-25T04:38:13.358922+0000","last_fullsized":"2023-03-25T04:38:13.358922+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:23.364593+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:39:23.364593+0000","last_peered":"2023-03-25T04:39:23.364593+0000","last_clean":"2023-03-25T04:39:23.364593+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:39:23.364593+0000","last_undegraded":"2023-03-25T04:39:23.364593+0000","last_fullsized":"2023-03-25T04:39:23.364593+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:45.608475+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:39:45.608475+0000","last_peered":"2023-03-25T04:39:45.608475+0000","last_clean":"2023-03-25T04:39:45.608475+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:39:45.608475+0000","last_undegraded":"2023-03-25T04:39:45.608475+0000","last_fullsized":"2023-03-25T04:39:45.608475+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":40,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:53.367398+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:39:53.367398+0000","last_peered":"2023-03-25T04:39:53.367398+0000","last_clean":"2023-03-25T04:39:53.367398+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:39:53.367398+0000","last_undegraded":"2023-03-25T04:39:53.367398+0000","last_fullsized":"2023-03-25T04:39:53.367398+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:53.367424+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:39:53.367424+0000","last_peered":"2023-03-25T04:39:53.367424+0000","last_clean":"2023-03-25T04:39:53.367424+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:39:53.367424+0000","last_undegraded":"2023-03-25T04:39:53.367424+0000","last_fullsized":"2023-03-25T04:39:53.367424+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117175,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:41:58 2023","interfaces":[{"interface":"back","average":{"1min":0.25800000000000001,"5min":0.26400000000000001,"15min":0.26800000000000002},"min":{"1min":0.17499999999999999,"5min":0.17499999999999999,"15min":0.16},"max":{"1min":0.39100000000000001,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.22900000000000001},{"interface":"front","average":{"1min":0.26300000000000001,"5min":0.27200000000000002,"15min":0.27400000000000002},"min":{"1min":0.20599999999999999,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.35699999999999998,"5min":0.46400000000000002,"15min":0.46400000000000002},"last":0.26400000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051765,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:41:52 2023","interfaces":[{"interface":"back","average":{"1min":0.26500000000000001,"5min":0.27200000000000002,"15min":0.26800000000000002},"min":{"1min":0.17000000000000001,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.39200000000000002,"5min":0.44,"15min":0.44},"last":0.23899999999999999},{"interface":"front","average":{"1min":0.28100000000000003,"5min":0.28599999999999998,"15min":0.28199999999999997},"min":{"1min":0.19400000000000001,"5min":0.19400000000000001,"15min":0.16600000000000001},"max":{"1min":0.41399999999999998,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.254}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:42:18.297+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:42:18.958+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:42:35.512 INFO:teuthology.orchestra.run.smithi070.stdout:
, 2023-03-25T04:42:35.512 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":1013,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":17,"modified":"2023-03-25T04:41:54.262744+0000","services":{}},"progress_events":{}}
, 2023-03-25T04:42:35.525 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:42:52.741 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":548,"stamp":"2023-03-25T04:42:52.273601+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002221"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":40,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:40:23.370056+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:40:23.370056+0000","last_peered":"2023-03-25T04:40:23.370056+0000","last_clean":"2023-03-25T04:40:23.370056+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:40:23.370056+0000","last_undegraded":"2023-03-25T04:40:23.370056+0000","last_fullsized":"2023-03-25T04:40:23.370056+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:40:23.370039+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:40:23.370039+0000","last_peered":"2023-03-25T04:40:23.370039+0000","last_clean":"2023-03-25T04:40:23.370039+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:40:23.370039+0000","last_undegraded":"2023-03-25T04:40:23.370039+0000","last_fullsized":"2023-03-25T04:40:23.370039+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:23.364593+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:39:23.364593+0000","last_peered":"2023-03-25T04:39:23.364593+0000","last_clean":"2023-03-25T04:39:23.364593+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:39:23.364593+0000","last_undegraded":"2023-03-25T04:39:23.364593+0000","last_fullsized":"2023-03-25T04:39:23.364593+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:45.608475+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:39:45.608475+0000","last_peered":"2023-03-25T04:39:45.608475+0000","last_clean":"2023-03-25T04:39:45.608475+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:39:45.608475+0000","last_undegraded":"2023-03-25T04:39:45.608475+0000","last_fullsized":"2023-03-25T04:39:45.608475+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":40,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:53.367398+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:39:53.367398+0000","last_peered":"2023-03-25T04:39:53.367398+0000","last_clean":"2023-03-25T04:39:53.367398+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:39:53.367398+0000","last_undegraded":"2023-03-25T04:39:53.367398+0000","last_fullsized":"2023-03-25T04:39:53.367398+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:53.367424+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:39:53.367424+0000","last_peered":"2023-03-25T04:39:53.367424+0000","last_clean":"2023-03-25T04:39:53.367424+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:39:53.367424+0000","last_undegraded":"2023-03-25T04:39:53.367424+0000","last_fullsized":"2023-03-25T04:39:53.367424+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117182,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:41:58 2023","interfaces":[{"interface":"back","average":{"1min":0.25800000000000001,"5min":0.26400000000000001,"15min":0.26800000000000002},"min":{"1min":0.17499999999999999,"5min":0.17499999999999999,"15min":0.16},"max":{"1min":0.39100000000000001,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.19600000000000001},{"interface":"front","average":{"1min":0.26300000000000001,"5min":0.27200000000000002,"15min":0.27400000000000002},"min":{"1min":0.20599999999999999,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.35699999999999998,"5min":0.46400000000000002,"15min":0.46400000000000002},"last":0.219}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051773,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:41:52 2023","interfaces":[{"interface":"back","average":{"1min":0.26500000000000001,"5min":0.27200000000000002,"15min":0.26800000000000002},"min":{"1min":0.17000000000000001,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.39200000000000002,"5min":0.44,"15min":0.44},"last":0.19},{"interface":"front","average":{"1min":0.28100000000000003,"5min":0.28599999999999998,"15min":0.28199999999999997},"min":{"1min":0.19400000000000001,"5min":0.19400000000000001,"15min":0.16600000000000001},"max":{"1min":0.41399999999999998,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.17599999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:42:52.742 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean

Ignoring malparsed backtrace: 2023-03-25T04:43:09.973 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:43:09.986 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":556,"stamp":"2023-03-25T04:43:08.276556+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002359"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":40,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:40:23.370056+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:40:23.370056+0000","last_peered":"2023-03-25T04:40:23.370056+0000","last_clean":"2023-03-25T04:40:23.370056+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:40:23.370056+0000","last_undegraded":"2023-03-25T04:40:23.370056+0000","last_fullsized":"2023-03-25T04:40:23.370056+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:40:23.370039+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:40:23.370039+0000","last_peered":"2023-03-25T04:40:23.370039+0000","last_clean":"2023-03-25T04:40:23.370039+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:40:23.370039+0000","last_undegraded":"2023-03-25T04:40:23.370039+0000","last_fullsized":"2023-03-25T04:40:23.370039+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:23.364593+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:39:23.364593+0000","last_peered":"2023-03-25T04:39:23.364593+0000","last_clean":"2023-03-25T04:39:23.364593+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:39:23.364593+0000","last_undegraded":"2023-03-25T04:39:23.364593+0000","last_fullsized":"2023-03-25T04:39:23.364593+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:45.608475+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:39:45.608475+0000","last_peered":"2023-03-25T04:39:45.608475+0000","last_clean":"2023-03-25T04:39:45.608475+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:39:45.608475+0000","last_undegraded":"2023-03-25T04:39:45.608475+0000","last_fullsized":"2023-03-25T04:39:45.608475+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":40,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:53.367398+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:39:53.367398+0000","last_peered":"2023-03-25T04:39:53.367398+0000","last_clean":"2023-03-25T04:39:53.367398+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:39:53.367398+0000","last_undegraded":"2023-03-25T04:39:53.367398+0000","last_fullsized":"2023-03-25T04:39:53.367398+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:53.367424+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:39:53.367424+0000","last_peered":"2023-03-25T04:39:53.367424+0000","last_clean":"2023-03-25T04:39:53.367424+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:39:53.367424+0000","last_undegraded":"2023-03-25T04:39:53.367424+0000","last_fullsized":"2023-03-25T04:39:53.367424+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117185,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:43:00 2023","interfaces":[{"interface":"back","average":{"1min":0.27400000000000002,"5min":0.26700000000000002,"15min":0.26600000000000001},"min":{"1min":0.19600000000000001,"5min":0.17499999999999999,"15min":0.16},"max":{"1min":0.42999999999999999,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.27800000000000002},{"interface":"front","average":{"1min":0.28199999999999997,"5min":0.27500000000000002,"15min":0.27300000000000002},"min":{"1min":0.185,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.47199999999999998,"5min":0.47199999999999998,"15min":0.47199999999999998},"last":0.247}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051776,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:42:54 2023","interfaces":[{"interface":"back","average":{"1min":0.26300000000000001,"5min":0.27000000000000002,"15min":0.26900000000000002},"min":{"1min":0.17399999999999999,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.438,"5min":0.44,"15min":0.44},"last":0.20100000000000001},{"interface":"front","average":{"1min":0.27400000000000002,"5min":0.28199999999999997,"15min":0.28299999999999997},"min":{"1min":0.17599999999999999,"5min":0.17599999999999999,"15min":0.16600000000000001},"max":{"1min":0.41599999999999998,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.23499999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:43:25.042+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:43:25.411+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:43:26.013+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:43:47.937+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:43:48.534+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:43:48.889+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:44:09.318 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:44:09.331 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":586,"stamp":"2023-03-25T04:44:08.288933+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002697"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":40,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:40:23.370056+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:40:23.370056+0000","last_peered":"2023-03-25T04:40:23.370056+0000","last_clean":"2023-03-25T04:40:23.370056+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:40:23.370056+0000","last_undegraded":"2023-03-25T04:40:23.370056+0000","last_fullsized":"2023-03-25T04:40:23.370056+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:40:23.370039+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:40:23.370039+0000","last_peered":"2023-03-25T04:40:23.370039+0000","last_clean":"2023-03-25T04:40:23.370039+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:40:23.370039+0000","last_undegraded":"2023-03-25T04:40:23.370039+0000","last_fullsized":"2023-03-25T04:40:23.370039+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:23.375106+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:41:23.375106+0000","last_peered":"2023-03-25T04:41:23.375106+0000","last_clean":"2023-03-25T04:41:23.375106+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:41:23.375106+0000","last_undegraded":"2023-03-25T04:41:23.375106+0000","last_fullsized":"2023-03-25T04:41:23.375106+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:50.618930+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:41:50.618930+0000","last_peered":"2023-03-25T04:41:50.618930+0000","last_clean":"2023-03-25T04:41:50.618930+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:41:50.618930+0000","last_undegraded":"2023-03-25T04:41:50.618930+0000","last_fullsized":"2023-03-25T04:41:50.618930+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":41,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:58.377821+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:41:58.377821+0000","last_peered":"2023-03-25T04:41:58.377821+0000","last_clean":"2023-03-25T04:41:58.377821+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:41:58.377821+0000","last_undegraded":"2023-03-25T04:41:58.377821+0000","last_fullsized":"2023-03-25T04:41:58.377821+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:58.377836+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:41:58.377836+0000","last_peered":"2023-03-25T04:41:58.377836+0000","last_clean":"2023-03-25T04:41:58.377836+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:41:58.377836+0000","last_undegraded":"2023-03-25T04:41:58.377836+0000","last_fullsized":"2023-03-25T04:41:58.377836+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117197,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:44:01 2023","interfaces":[{"interface":"back","average":{"1min":0.27200000000000002,"5min":0.27000000000000002,"15min":0.26400000000000001},"min":{"1min":0.20399999999999999,"5min":0.17499999999999999,"15min":0.16},"max":{"1min":0.38400000000000001,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.245},{"interface":"front","average":{"1min":0.27600000000000002,"5min":0.27800000000000002,"15min":0.27100000000000002},"min":{"1min":0.23100000000000001,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.40400000000000003,"5min":0.47199999999999998,"15min":0.47199999999999998},"last":0.27500000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051788,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:43:55 2023","interfaces":[{"interface":"back","average":{"1min":0.26100000000000001,"5min":0.26700000000000002,"15min":0.27000000000000002},"min":{"1min":0.17799999999999999,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.374,"5min":0.44,"15min":0.44},"last":0.30299999999999999},{"interface":"front","average":{"1min":0.28299999999999997,"5min":0.28100000000000003,"15min":0.28399999999999997},"min":{"1min":0.20899999999999999,"5min":0.17599999999999999,"15min":0.16600000000000001},"max":{"1min":0.40100000000000002,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.27800000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:44:28.923 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":1126,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":19,"modified":"2023-03-25T04:43:20.278959+0000","services":{}},"progress_events":{}}
, 2023-03-25T04:44:28.936 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now
, 2023-03-25T04:44:28.937 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace: 2023-03-25T04:44:48.786 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:44:48.800 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":606,"stamp":"2023-03-25T04:44:48.296805+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002182"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":41,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:42:34.382196+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:42:34.382196+0000","last_peered":"2023-03-25T04:42:34.382196+0000","last_clean":"2023-03-25T04:42:34.382196+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:42:34.382196+0000","last_undegraded":"2023-03-25T04:42:34.382196+0000","last_fullsized":"2023-03-25T04:42:34.382196+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:42:29.381744+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:42:29.381744+0000","last_peered":"2023-03-25T04:42:29.381744+0000","last_clean":"2023-03-25T04:42:29.381744+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:42:29.381744+0000","last_undegraded":"2023-03-25T04:42:29.381744+0000","last_fullsized":"2023-03-25T04:42:29.381744+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:23.375106+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:41:23.375106+0000","last_peered":"2023-03-25T04:41:23.375106+0000","last_clean":"2023-03-25T04:41:23.375106+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:41:23.375106+0000","last_undegraded":"2023-03-25T04:41:23.375106+0000","last_fullsized":"2023-03-25T04:41:23.375106+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:50.618930+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:41:50.618930+0000","last_peered":"2023-03-25T04:41:50.618930+0000","last_clean":"2023-03-25T04:41:50.618930+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:41:50.618930+0000","last_undegraded":"2023-03-25T04:41:50.618930+0000","last_fullsized":"2023-03-25T04:41:50.618930+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":41,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:58.377821+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:41:58.377821+0000","last_peered":"2023-03-25T04:41:58.377821+0000","last_clean":"2023-03-25T04:41:58.377821+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:41:58.377821+0000","last_undegraded":"2023-03-25T04:41:58.377821+0000","last_fullsized":"2023-03-25T04:41:58.377821+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:58.377836+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:41:58.377836+0000","last_peered":"2023-03-25T04:41:58.377836+0000","last_clean":"2023-03-25T04:41:58.377836+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:41:58.377836+0000","last_undegraded":"2023-03-25T04:41:58.377836+0000","last_fullsized":"2023-03-25T04:41:58.377836+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117206,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:44:01 2023","interfaces":[{"interface":"back","average":{"1min":0.27200000000000002,"5min":0.27000000000000002,"15min":0.26400000000000001},"min":{"1min":0.20399999999999999,"5min":0.17499999999999999,"15min":0.16},"max":{"1min":0.38400000000000001,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.216},{"interface":"front","average":{"1min":0.27600000000000002,"5min":0.27800000000000002,"15min":0.27100000000000002},"min":{"1min":0.23100000000000001,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.40400000000000003,"5min":0.47199999999999998,"15min":0.47199999999999998},"last":0.23999999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051796,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:43:55 2023","interfaces":[{"interface":"back","average":{"1min":0.26100000000000001,"5min":0.26700000000000002,"15min":0.27000000000000002},"min":{"1min":0.17799999999999999,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.374,"5min":0.44,"15min":0.44},"last":0.25600000000000001},{"interface":"front","average":{"1min":0.28299999999999997,"5min":0.28100000000000003,"15min":0.28399999999999997},"min":{"1min":0.20899999999999999,"5min":0.17599999999999999,"15min":0.16600000000000001},"max":{"1min":0.40100000000000002,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.28000000000000003}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:45:05.762 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:45:06.073 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all

Ignoring malparsed backtrace: 2023-03-25T04:45:25.658 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":624,"stamp":"2023-03-25T04:45:24.304294+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002366"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":41,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:42:34.382196+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:42:34.382196+0000","last_peered":"2023-03-25T04:42:34.382196+0000","last_clean":"2023-03-25T04:42:34.382196+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:42:34.382196+0000","last_undegraded":"2023-03-25T04:42:34.382196+0000","last_fullsized":"2023-03-25T04:42:34.382196+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:42:29.381744+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:42:29.381744+0000","last_peered":"2023-03-25T04:42:29.381744+0000","last_clean":"2023-03-25T04:42:29.381744+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:42:29.381744+0000","last_undegraded":"2023-03-25T04:42:29.381744+0000","last_fullsized":"2023-03-25T04:42:29.381744+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:23.375106+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:41:23.375106+0000","last_peered":"2023-03-25T04:41:23.375106+0000","last_clean":"2023-03-25T04:41:23.375106+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:41:23.375106+0000","last_undegraded":"2023-03-25T04:41:23.375106+0000","last_fullsized":"2023-03-25T04:41:23.375106+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:50.618930+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:41:50.618930+0000","last_peered":"2023-03-25T04:41:50.618930+0000","last_clean":"2023-03-25T04:41:50.618930+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:41:50.618930+0000","last_undegraded":"2023-03-25T04:41:50.618930+0000","last_fullsized":"2023-03-25T04:41:50.618930+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":41,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:58.377821+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:41:58.377821+0000","last_peered":"2023-03-25T04:41:58.377821+0000","last_clean":"2023-03-25T04:41:58.377821+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:41:58.377821+0000","last_undegraded":"2023-03-25T04:41:58.377821+0000","last_fullsized":"2023-03-25T04:41:58.377821+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:58.377836+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:41:58.377836+0000","last_peered":"2023-03-25T04:41:58.377836+0000","last_clean":"2023-03-25T04:41:58.377836+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:41:58.377836+0000","last_undegraded":"2023-03-25T04:41:58.377836+0000","last_fullsized":"2023-03-25T04:41:58.377836+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117213,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:45:01 2023","interfaces":[{"interface":"back","average":{"1min":0.251,"5min":0.26800000000000002,"15min":0.26400000000000001},"min":{"1min":0.183,"5min":0.17499999999999999,"15min":0.16},"max":{"1min":0.41499999999999998,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.26900000000000002},{"interface":"front","average":{"1min":0.28000000000000003,"5min":0.27900000000000003,"15min":0.27100000000000002},"min":{"1min":0.20999999999999999,"5min":0.185,"15min":0.17499999999999999},"max":{"1min":0.44400000000000001,"5min":0.47199999999999998,"15min":0.47199999999999998},"last":0.27900000000000003}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051803,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:44:55 2023","interfaces":[{"interface":"back","average":{"1min":0.252,"5min":0.26500000000000001,"15min":0.26900000000000002},"min":{"1min":0.183,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.39700000000000002,"5min":0.44,"15min":0.44},"last":0.20699999999999999},{"interface":"front","average":{"1min":0.27000000000000002,"5min":0.28100000000000003,"15min":0.28299999999999997},"min":{"1min":0.20300000000000001,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":0.42099999999999999,"5min":0.47599999999999998,"15min":0.48599999999999999},"last":0.23599999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:45:25.659 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:45:25.969 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:45:42.735+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:45:42.744+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:45:42.897 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace: 2023-03-25T04:46:00.183 INFO:teuthology.orchestra.run.smithi070.stdout:
, 2023-03-25T04:46:00.183 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":1217,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":20,"modified":"2023-03-25T04:45:42.307700+0000","services":{"mgr":{"daemons":{"summary":"","x":{"start_epoch":0,"start_stamp":"0.000000","gid":0,"addr":"(unrecognized address family 0)/0","metadata":{},"task_status":{}}}}}},"progress_events":{}}
, 2023-03-25T04:46:00.197 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now

Ignoring malparsed backtrace: 2023-03-25T04:46:19.910+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:46:20.039 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all

Ignoring malparsed backtrace: 2023-03-25T04:46:39.611 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}
, 2023-03-25T04:46:39.611 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json
, 2023-03-25T04:46:39.613+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:46:58.485+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:46:58.764+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:46:59.519+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:47:14.132 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:47:14.146 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":678,"stamp":"2023-03-25T04:47:12.325044+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002571"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":42,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:44.393426+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:44:44.393426+0000","last_peered":"2023-03-25T04:44:44.393426+0000","last_clean":"2023-03-25T04:44:44.393426+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:44:44.393426+0000","last_undegraded":"2023-03-25T04:44:44.393426+0000","last_fullsized":"2023-03-25T04:44:44.393426+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:39.393044+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:44:39.393044+0000","last_peered":"2023-03-25T04:44:39.393044+0000","last_clean":"2023-03-25T04:44:39.393044+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:44:39.393044+0000","last_undegraded":"2023-03-25T04:44:39.393044+0000","last_fullsized":"2023-03-25T04:44:39.393044+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:43:24.386177+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:43:24.386177+0000","last_peered":"2023-03-25T04:43:24.386177+0000","last_clean":"2023-03-25T04:43:24.386177+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:43:24.386177+0000","last_undegraded":"2023-03-25T04:43:24.386177+0000","last_fullsized":"2023-03-25T04:43:24.386177+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:43:55.629798+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:43:55.629798+0000","last_peered":"2023-03-25T04:43:55.629798+0000","last_clean":"2023-03-25T04:43:55.629798+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:43:55.629798+0000","last_undegraded":"2023-03-25T04:43:55.629798+0000","last_fullsized":"2023-03-25T04:43:55.629798+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":42,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:04.389904+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:44:04.389904+0000","last_peered":"2023-03-25T04:44:04.389904+0000","last_clean":"2023-03-25T04:44:04.389904+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:44:04.389904+0000","last_undegraded":"2023-03-25T04:44:04.389904+0000","last_fullsized":"2023-03-25T04:44:04.389904+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:04.389923+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:44:04.389923+0000","last_peered":"2023-03-25T04:44:04.389923+0000","last_clean":"2023-03-25T04:44:04.389923+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:44:04.389923+0000","last_undegraded":"2023-03-25T04:44:04.389923+0000","last_fullsized":"2023-03-25T04:44:04.389923+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117235,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:47:05 2023","interfaces":[{"interface":"back","average":{"1min":0.27800000000000002,"5min":0.27000000000000002,"15min":0.26300000000000001},"min":{"1min":0.20100000000000001,"5min":0.16900000000000001,"15min":0.16},"max":{"1min":0.39200000000000002,"5min":0.42999999999999999,"15min":0.45600000000000002},"last":0.379},{"interface":"front","average":{"1min":0.28399999999999997,"5min":0.28100000000000003,"15min":0.27200000000000002},"min":{"1min":0.17799999999999999,"5min":0.17799999999999999,"15min":0.17699999999999999},"max":{"1min":0.42599999999999999,"5min":0.47199999999999998,"15min":0.47199999999999998},"last":0.39800000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051825,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:47:01 2023","interfaces":[{"interface":"back","average":{"1min":0.27300000000000002,"5min":0.26100000000000001,"15min":0.27000000000000002},"min":{"1min":0.186,"5min":0.17399999999999999,"15min":0.17000000000000001},"max":{"1min":0.40400000000000003,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.23899999999999999},{"interface":"front","average":{"1min":0.28699999999999998,"5min":0.27500000000000002,"15min":0.28299999999999997},"min":{"1min":0.186,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.48599999999999999},"last":0.19600000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:47:31.198 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json
, 2023-03-25T04:47:31.411+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:47:31.789 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:47:49.509 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":696,"stamp":"2023-03-25T04:47:48.332093+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002697"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":42,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:44.393426+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:44:44.393426+0000","last_peered":"2023-03-25T04:44:44.393426+0000","last_clean":"2023-03-25T04:44:44.393426+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:44:44.393426+0000","last_undegraded":"2023-03-25T04:44:44.393426+0000","last_fullsized":"2023-03-25T04:44:44.393426+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:39.393044+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:44:39.393044+0000","last_peered":"2023-03-25T04:44:39.393044+0000","last_clean":"2023-03-25T04:44:39.393044+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:44:39.393044+0000","last_undegraded":"2023-03-25T04:44:39.393044+0000","last_fullsized":"2023-03-25T04:44:39.393044+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:45:24.396717+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:45:24.396717+0000","last_peered":"2023-03-25T04:45:24.396717+0000","last_clean":"2023-03-25T04:45:24.396717+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:45:24.396717+0000","last_undegraded":"2023-03-25T04:45:24.396717+0000","last_fullsized":"2023-03-25T04:45:24.396717+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:43:55.629798+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:43:55.629798+0000","last_peered":"2023-03-25T04:43:55.629798+0000","last_clean":"2023-03-25T04:43:55.629798+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:43:55.629798+0000","last_undegraded":"2023-03-25T04:43:55.629798+0000","last_fullsized":"2023-03-25T04:43:55.629798+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":42,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:04.389904+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:44:04.389904+0000","last_peered":"2023-03-25T04:44:04.389904+0000","last_clean":"2023-03-25T04:44:04.389904+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:44:04.389904+0000","last_undegraded":"2023-03-25T04:44:04.389904+0000","last_fullsized":"2023-03-25T04:44:04.389904+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:04.389923+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:44:04.389923+0000","last_peered":"2023-03-25T04:44:04.389923+0000","last_clean":"2023-03-25T04:44:04.389923+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:44:04.389923+0000","last_undegraded":"2023-03-25T04:44:04.389923+0000","last_fullsized":"2023-03-25T04:44:04.389923+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117242,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:47:05 2023","interfaces":[{"interface":"back","average":{"1min":0.27800000000000002,"5min":0.27000000000000002,"15min":0.26300000000000001},"min":{"1min":0.20100000000000001,"5min":0.16900000000000001,"15min":0.16},"max":{"1min":0.39200000000000002,"5min":0.42999999999999999,"15min":0.45600000000000002},"last":0.40500000000000003},{"interface":"front","average":{"1min":0.28399999999999997,"5min":0.28100000000000003,"15min":0.27200000000000002},"min":{"1min":0.17799999999999999,"5min":0.17799999999999999,"15min":0.17699999999999999},"max":{"1min":0.42599999999999999,"5min":0.47199999999999998,"15min":0.47199999999999998},"last":0.42399999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051834,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:47:01 2023","interfaces":[{"interface":"back","average":{"1min":0.27300000000000002,"5min":0.26100000000000001,"15min":0.27000000000000002},"min":{"1min":0.186,"5min":0.17399999999999999,"15min":0.17000000000000001},"max":{"1min":0.40400000000000003,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.25600000000000001},{"interface":"front","average":{"1min":0.28699999999999998,"5min":0.27500000000000002,"15min":0.28299999999999997},"min":{"1min":0.186,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.48599999999999999},"last":0.22800000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:47:49.510 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace:     ctx.managers[config['cluster']].wait_for_clean()
,   File "/home/teuthworker/src/github.com_ceph_ceph-c_511015c23eef0684a64171b710e432448df2b7c7/qa/tasks/ceph_manager.py", line 2747, in wait_for_clean
,     assert time.time() - start < timeout, \

Ignoring malparsed backtrace:  ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
,  1: /lib64/libpthread.so.0(+0x12cf0) [0x7f1453c2fcf0]
,  2: __pthread_mutex_lock()

Ignoring malparsed backtrace: 2023-03-25T04:28:05.172 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":142,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":5,"modified":"2023-03-25T04:27:12.092098+0000","services":{}},"progress_events":{}}
, 2023-03-25T04:28:05.185 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now
, 2023-03-25T04:28:05.186 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace: 2023-03-25T04:28:28.360 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":116,"stamp":"2023-03-25T04:28:28.108007+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002578"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.314661+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:27:45.314661+0000","last_peered":"2023-03-25T04:27:45.314661+0000","last_clean":"2023-03-25T04:27:45.314661+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:27:45.314661+0000","last_undegraded":"2023-03-25T04:27:45.314661+0000","last_fullsized":"2023-03-25T04:27:45.314661+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":25,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.310745+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:27:44.310745+0000","last_peered":"2023-03-25T04:27:44.310745+0000","last_clean":"2023-03-25T04:27:44.310745+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:27:44.310745+0000","last_undegraded":"2023-03-25T04:27:44.310745+0000","last_fullsized":"2023-03-25T04:27:44.310745+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":25,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.310650+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:27:44.310650+0000","last_peered":"2023-03-25T04:27:44.310650+0000","last_clean":"2023-03-25T04:27:44.310650+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:27:44.310650+0000","last_undegraded":"2023-03-25T04:27:44.310650+0000","last_fullsized":"2023-03-25T04:27:44.310650+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":26,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.312962+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:27:44.312962+0000","last_peered":"2023-03-25T04:27:44.312962+0000","last_clean":"2023-03-25T04:27:44.312962+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:27:44.312962+0000","last_undegraded":"2023-03-25T04:27:44.312962+0000","last_fullsized":"2023-03-25T04:27:44.312962+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315064+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:27:45.315064+0000","last_peered":"2023-03-25T04:27:45.315064+0000","last_clean":"2023-03-25T04:27:45.315064+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:27:45.315064+0000","last_undegraded":"2023-03-25T04:27:45.315064+0000","last_fullsized":"2023-03-25T04:27:45.315064+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":25,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.310707+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:27:44.310707+0000","last_peered":"2023-03-25T04:27:44.310707+0000","last_clean":"2023-03-25T04:27:44.310707+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:27:44.310707+0000","last_undegraded":"2023-03-25T04:27:44.310707+0000","last_fullsized":"2023-03-25T04:27:44.310707+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117007,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.375}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051597,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.186}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:28:28.362 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean
, 2023-03-25T04:28:28.362 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:28:45.655 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":124,"stamp":"2023-03-25T04:28:44.111265+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002492"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.314661+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:27:45.314661+0000","last_peered":"2023-03-25T04:27:45.314661+0000","last_clean":"2023-03-25T04:27:45.314661+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:27:45.314661+0000","last_undegraded":"2023-03-25T04:27:45.314661+0000","last_fullsized":"2023-03-25T04:27:45.314661+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":25,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.310745+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:27:44.310745+0000","last_peered":"2023-03-25T04:27:44.310745+0000","last_clean":"2023-03-25T04:27:44.310745+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:27:44.310745+0000","last_undegraded":"2023-03-25T04:27:44.310745+0000","last_fullsized":"2023-03-25T04:27:44.310745+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":25,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.310650+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:27:44.310650+0000","last_peered":"2023-03-25T04:27:44.310650+0000","last_clean":"2023-03-25T04:27:44.310650+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:27:44.310650+0000","last_undegraded":"2023-03-25T04:27:44.310650+0000","last_fullsized":"2023-03-25T04:27:44.310650+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":26,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.312962+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:27:44.312962+0000","last_peered":"2023-03-25T04:27:44.312962+0000","last_clean":"2023-03-25T04:27:44.312962+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:27:44.312962+0000","last_undegraded":"2023-03-25T04:27:44.312962+0000","last_fullsized":"2023-03-25T04:27:44.312962+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315064+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:27:45.315064+0000","last_peered":"2023-03-25T04:27:45.315064+0000","last_clean":"2023-03-25T04:27:45.315064+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:27:45.315064+0000","last_undegraded":"2023-03-25T04:27:45.315064+0000","last_fullsized":"2023-03-25T04:27:45.315064+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":25,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.310707+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:27:44.310707+0000","last_peered":"2023-03-25T04:27:44.310707+0000","last_clean":"2023-03-25T04:27:44.310707+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:27:44.310707+0000","last_undegraded":"2023-03-25T04:27:44.310707+0000","last_fullsized":"2023-03-25T04:27:44.310707+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117011,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:28:31 2023","interfaces":[{"interface":"back","average":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.36899999999999999},{"interface":"front","average":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"min":{"1min":0.19,"5min":0.19,"15min":0.19},"max":{"1min":0.46300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.39800000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051601,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:28:24 2023","interfaces":[{"interface":"back","average":{"1min":0.249,"5min":0.249,"15min":0.249},"min":{"1min":0.17699999999999999,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"last":0.254},{"interface":"front","average":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"min":{"1min":0.20200000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"last":0.312}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:28:45.656 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace: 2023-03-25T04:29:05.171 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}
, 2023-03-25T04:29:05.171 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json
, 2023-03-25T04:29:05.561 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:29:22.465 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:29:22.773 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all

Ignoring malparsed backtrace: 2023-03-25T04:29:42.405 INFO:teuthology.orchestra.run.smithi070.stdout:
, 2023-03-25T04:29:42.406 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":240,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":9,"modified":"2023-03-25T04:29:32.120358+0000","services":{"mon":{"daemons":{"summary":"","a":{"start_epoch":0,"start_stamp":"0.000000","gid":0,"addr":"(unrecognized address family 0)/0","metadata":{},"task_status":{}}}}}},"progress_events":{}}
, 2023-03-25T04:29:42.419 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:29:59.627 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":161,"stamp":"2023-03-25T04:29:58.125186+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002398"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315468+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:27:45.315468+0000","last_peered":"2023-03-25T04:27:45.315468+0000","last_clean":"2023-03-25T04:27:45.315468+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:27:45.315468+0000","last_undegraded":"2023-03-25T04:27:45.315468+0000","last_fullsized":"2023-03-25T04:27:45.315468+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312845+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:27:45.312845+0000","last_peered":"2023-03-25T04:27:45.312845+0000","last_clean":"2023-03-25T04:27:45.312845+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:27:45.312845+0000","last_undegraded":"2023-03-25T04:27:45.312845+0000","last_fullsized":"2023-03-25T04:27:45.312845+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312882+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:27:45.312882+0000","last_peered":"2023-03-25T04:27:45.312882+0000","last_clean":"2023-03-25T04:27:45.312882+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:27:45.312882+0000","last_undegraded":"2023-03-25T04:27:45.312882+0000","last_fullsized":"2023-03-25T04:27:45.312882+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.311730+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:27:45.311730+0000","last_peered":"2023-03-25T04:27:45.311730+0000","last_clean":"2023-03-25T04:27:45.311730+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:27:45.311730+0000","last_undegraded":"2023-03-25T04:27:45.311730+0000","last_fullsized":"2023-03-25T04:27:45.311730+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315735+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:27:45.315735+0000","last_peered":"2023-03-25T04:27:45.315735+0000","last_clean":"2023-03-25T04:27:45.315735+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:27:45.315735+0000","last_undegraded":"2023-03-25T04:27:45.315735+0000","last_fullsized":"2023-03-25T04:27:45.315735+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312833+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:27:45.312833+0000","last_peered":"2023-03-25T04:27:45.312833+0000","last_clean":"2023-03-25T04:27:45.312833+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:27:45.312833+0000","last_undegraded":"2023-03-25T04:27:45.312833+0000","last_fullsized":"2023-03-25T04:27:45.312833+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117026,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:29:33 2023","interfaces":[{"interface":"back","average":{"1min":0.26300000000000001,"5min":0.29099999999999998,"15min":0.29599999999999999},"min":{"1min":0.191,"5min":0.191,"15min":0.191},"max":{"1min":0.371,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.20499999999999999},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.29599999999999999,"15min":0.29999999999999999},"min":{"1min":0.21099999999999999,"5min":0.19,"15min":0.19},"max":{"1min":0.39800000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.19500000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051616,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:29:29 2023","interfaces":[{"interface":"back","average":{"1min":0.27200000000000002,"5min":0.253,"15min":0.25},"min":{"1min":0.183,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"last":0.26400000000000001},{"interface":"front","average":{"1min":0.28499999999999998,"5min":0.26800000000000002,"15min":0.26500000000000001},"min":{"1min":0.16600000000000001,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"last":0.29899999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:29:59.629 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean

Ignoring malparsed backtrace: 2023-03-25T04:30:16.889 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:30:16.903 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":170,"stamp":"2023-03-25T04:30:16.128427+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002222"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315468+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:27:45.315468+0000","last_peered":"2023-03-25T04:27:45.315468+0000","last_clean":"2023-03-25T04:27:45.315468+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:27:45.315468+0000","last_undegraded":"2023-03-25T04:27:45.315468+0000","last_fullsized":"2023-03-25T04:27:45.315468+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312845+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:27:45.312845+0000","last_peered":"2023-03-25T04:27:45.312845+0000","last_clean":"2023-03-25T04:27:45.312845+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:27:45.312845+0000","last_undegraded":"2023-03-25T04:27:45.312845+0000","last_fullsized":"2023-03-25T04:27:45.312845+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312882+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:27:45.312882+0000","last_peered":"2023-03-25T04:27:45.312882+0000","last_clean":"2023-03-25T04:27:45.312882+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:27:45.312882+0000","last_undegraded":"2023-03-25T04:27:45.312882+0000","last_fullsized":"2023-03-25T04:27:45.312882+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.311730+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:27:45.311730+0000","last_peered":"2023-03-25T04:27:45.311730+0000","last_clean":"2023-03-25T04:27:45.311730+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:27:45.311730+0000","last_undegraded":"2023-03-25T04:27:45.311730+0000","last_fullsized":"2023-03-25T04:27:45.311730+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315735+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:27:45.315735+0000","last_peered":"2023-03-25T04:27:45.315735+0000","last_clean":"2023-03-25T04:27:45.315735+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:27:45.315735+0000","last_undegraded":"2023-03-25T04:27:45.315735+0000","last_fullsized":"2023-03-25T04:27:45.315735+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312833+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:27:45.312833+0000","last_peered":"2023-03-25T04:27:45.312833+0000","last_clean":"2023-03-25T04:27:45.312833+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:27:45.312833+0000","last_undegraded":"2023-03-25T04:27:45.312833+0000","last_fullsized":"2023-03-25T04:27:45.312833+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117030,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:29:33 2023","interfaces":[{"interface":"back","average":{"1min":0.26300000000000001,"5min":0.29099999999999998,"15min":0.29599999999999999},"min":{"1min":0.191,"5min":0.191,"15min":0.191},"max":{"1min":0.371,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.245},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.29599999999999999,"15min":0.29999999999999999},"min":{"1min":0.21099999999999999,"5min":0.19,"15min":0.19},"max":{"1min":0.39800000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.20899999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051620,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:29:29 2023","interfaces":[{"interface":"back","average":{"1min":0.27200000000000002,"5min":0.253,"15min":0.25},"min":{"1min":0.183,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"last":0.19400000000000001},{"interface":"front","average":{"1min":0.28499999999999998,"5min":0.26800000000000002,"15min":0.26500000000000001},"min":{"1min":0.16600000000000001,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"last":0.23599999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:30:31.984+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:30:32.201+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:30:32.981+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)

Ignoring malparsed backtrace: 2023-03-25T04:30:48.449 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now
, 2023-03-25T04:30:48.450 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:30:48.760 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:31:08.342 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":196,"stamp":"2023-03-25T04:31:08.139018+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002431"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315468+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:27:45.315468+0000","last_peered":"2023-03-25T04:27:45.315468+0000","last_clean":"2023-03-25T04:27:45.315468+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:27:45.315468+0000","last_undegraded":"2023-03-25T04:27:45.315468+0000","last_fullsized":"2023-03-25T04:27:45.315468+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312845+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:27:45.312845+0000","last_peered":"2023-03-25T04:27:45.312845+0000","last_clean":"2023-03-25T04:27:45.312845+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:27:45.312845+0000","last_undegraded":"2023-03-25T04:27:45.312845+0000","last_fullsized":"2023-03-25T04:27:45.312845+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312882+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:27:45.312882+0000","last_peered":"2023-03-25T04:27:45.312882+0000","last_clean":"2023-03-25T04:27:45.312882+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:27:45.312882+0000","last_undegraded":"2023-03-25T04:27:45.312882+0000","last_fullsized":"2023-03-25T04:27:45.312882+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.311730+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:27:45.311730+0000","last_peered":"2023-03-25T04:27:45.311730+0000","last_clean":"2023-03-25T04:27:45.311730+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:27:45.311730+0000","last_undegraded":"2023-03-25T04:27:45.311730+0000","last_fullsized":"2023-03-25T04:27:45.311730+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315735+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:27:45.315735+0000","last_peered":"2023-03-25T04:27:45.315735+0000","last_clean":"2023-03-25T04:27:45.315735+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:27:45.315735+0000","last_undegraded":"2023-03-25T04:27:45.315735+0000","last_fullsized":"2023-03-25T04:27:45.315735+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312833+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:27:45.312833+0000","last_peered":"2023-03-25T04:27:45.312833+0000","last_clean":"2023-03-25T04:27:45.312833+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:27:45.312833+0000","last_undegraded":"2023-03-25T04:27:45.312833+0000","last_fullsized":"2023-03-25T04:27:45.312833+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117040,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:30:37 2023","interfaces":[{"interface":"back","average":{"1min":0.28799999999999998,"5min":0.28899999999999998,"15min":0.29499999999999998},"min":{"1min":0.20200000000000001,"5min":0.191,"15min":0.191},"max":{"1min":0.433,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.29799999999999999},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.28999999999999998,"15min":0.29799999999999999},"min":{"1min":0.17499999999999999,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":0.40300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.25700000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051630,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:30:32 2023","interfaces":[{"interface":"back","average":{"1min":0.25600000000000001,"5min":0.255,"15min":0.251},"min":{"1min":0.19400000000000001,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.35699999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"last":0.22800000000000001},{"interface":"front","average":{"1min":0.28699999999999998,"5min":0.27200000000000002,"15min":0.26600000000000001},"min":{"1min":0.20599999999999999,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.39400000000000002,"5min":0.42899999999999999,"15min":0.42899999999999999},"last":0.24099999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:31:08.343 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace: 2023-03-25T04:31:24.283+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:31:25.192+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:31:25.295+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)

Ignoring malparsed backtrace: 2023-03-25T04:31:40.257 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:31:40.271 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":212,"stamp":"2023-03-25T04:31:40.144732+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002208"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315468+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:27:45.315468+0000","last_peered":"2023-03-25T04:27:45.315468+0000","last_clean":"2023-03-25T04:27:45.315468+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:27:45.315468+0000","last_undegraded":"2023-03-25T04:27:45.315468+0000","last_fullsized":"2023-03-25T04:27:45.315468+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312845+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:27:45.312845+0000","last_peered":"2023-03-25T04:27:45.312845+0000","last_clean":"2023-03-25T04:27:45.312845+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:27:45.312845+0000","last_undegraded":"2023-03-25T04:27:45.312845+0000","last_fullsized":"2023-03-25T04:27:45.312845+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:23.308607+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:29:23.308607+0000","last_peered":"2023-03-25T04:29:23.308607+0000","last_clean":"2023-03-25T04:29:23.308607+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:29:23.308607+0000","last_undegraded":"2023-03-25T04:29:23.308607+0000","last_fullsized":"2023-03-25T04:29:23.308607+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":28,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:20.554710+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:29:20.554710+0000","last_peered":"2023-03-25T04:29:20.554710+0000","last_clean":"2023-03-25T04:29:20.554710+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:29:20.554710+0000","last_undegraded":"2023-03-25T04:29:20.554710+0000","last_fullsized":"2023-03-25T04:29:20.554710+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309216+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:29:28.309216+0000","last_peered":"2023-03-25T04:29:28.309216+0000","last_clean":"2023-03-25T04:29:28.309216+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:29:28.309216+0000","last_undegraded":"2023-03-25T04:29:28.309216+0000","last_fullsized":"2023-03-25T04:29:28.309216+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309233+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:29:28.309233+0000","last_peered":"2023-03-25T04:29:28.309233+0000","last_clean":"2023-03-25T04:29:28.309233+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:29:28.309233+0000","last_undegraded":"2023-03-25T04:29:28.309233+0000","last_fullsized":"2023-03-25T04:29:28.309233+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117046,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:30:37 2023","interfaces":[{"interface":"back","average":{"1min":0.28799999999999998,"5min":0.28899999999999998,"15min":0.29499999999999998},"min":{"1min":0.20200000000000001,"5min":0.191,"15min":0.191},"max":{"1min":0.433,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.216},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.28999999999999998,"15min":0.29799999999999999},"min":{"1min":0.17499999999999999,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":0.40300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.24299999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051637,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:31:35 2023","interfaces":[{"interface":"back","average":{"1min":0.26100000000000001,"5min":0.25700000000000001,"15min":0.251},"min":{"1min":0.19,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.35899999999999999,"5min":0.41599999999999998,"15min":0.41599999999999998},"last":0.221},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.27400000000000002,"15min":0.26700000000000002},"min":{"1min":0.20200000000000001,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.374,"5min":0.42899999999999999,"15min":0.42899999999999999},"last":0.24099999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:32:01.906+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:32:02.388+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:32:02.794 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:32:22.326 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":233,"stamp":"2023-03-25T04:32:22.153289+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002545"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:33.309748+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:29:33.309748+0000","last_peered":"2023-03-25T04:29:33.309748+0000","last_clean":"2023-03-25T04:29:33.309748+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:29:33.309748+0000","last_undegraded":"2023-03-25T04:29:33.309748+0000","last_fullsized":"2023-03-25T04:29:33.309748+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:33.309729+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:29:33.309729+0000","last_peered":"2023-03-25T04:29:33.309729+0000","last_clean":"2023-03-25T04:29:33.309729+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:29:33.309729+0000","last_undegraded":"2023-03-25T04:29:33.309729+0000","last_fullsized":"2023-03-25T04:29:33.309729+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:23.308607+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:29:23.308607+0000","last_peered":"2023-03-25T04:29:23.308607+0000","last_clean":"2023-03-25T04:29:23.308607+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:29:23.308607+0000","last_undegraded":"2023-03-25T04:29:23.308607+0000","last_fullsized":"2023-03-25T04:29:23.308607+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":28,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:20.554710+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:29:20.554710+0000","last_peered":"2023-03-25T04:29:20.554710+0000","last_clean":"2023-03-25T04:29:20.554710+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:29:20.554710+0000","last_undegraded":"2023-03-25T04:29:20.554710+0000","last_fullsized":"2023-03-25T04:29:20.554710+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309216+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:29:28.309216+0000","last_peered":"2023-03-25T04:29:28.309216+0000","last_clean":"2023-03-25T04:29:28.309216+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:29:28.309216+0000","last_undegraded":"2023-03-25T04:29:28.309216+0000","last_fullsized":"2023-03-25T04:29:28.309216+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309233+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:29:28.309233+0000","last_peered":"2023-03-25T04:29:28.309233+0000","last_clean":"2023-03-25T04:29:28.309233+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:29:28.309233+0000","last_undegraded":"2023-03-25T04:29:28.309233+0000","last_fullsized":"2023-03-25T04:29:28.309233+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117055,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:31:42 2023","interfaces":[{"interface":"back","average":{"1min":0.27500000000000002,"5min":0.28399999999999997,"15min":0.29399999999999998},"min":{"1min":0.19800000000000001,"5min":0.191,"15min":0.191},"max":{"1min":0.36899999999999999,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.32300000000000001},{"interface":"front","average":{"1min":0.27900000000000003,"5min":0.28499999999999998,"15min":0.29599999999999999},"min":{"1min":0.216,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":0.40799999999999997,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.27100000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051646,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:31:35 2023","interfaces":[{"interface":"back","average":{"1min":0.26100000000000001,"5min":0.25700000000000001,"15min":0.251},"min":{"1min":0.19,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.35899999999999999,"5min":0.41599999999999998,"15min":0.41599999999999998},"last":0.25600000000000001},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.27400000000000002,"15min":0.26700000000000002},"min":{"1min":0.20200000000000001,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.374,"5min":0.42899999999999999,"15min":0.42899999999999999},"last":0.29199999999999998}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:32:22.327 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean
, 2023-03-25T04:32:22.327 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}

Ignoring malparsed backtrace: 2023-03-25T04:32:39.836 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:32:39.849 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":241,"stamp":"2023-03-25T04:32:38.156312+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002346"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:33.309748+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:29:33.309748+0000","last_peered":"2023-03-25T04:29:33.309748+0000","last_clean":"2023-03-25T04:29:33.309748+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:29:33.309748+0000","last_undegraded":"2023-03-25T04:29:33.309748+0000","last_fullsized":"2023-03-25T04:29:33.309748+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:33.309729+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:29:33.309729+0000","last_peered":"2023-03-25T04:29:33.309729+0000","last_clean":"2023-03-25T04:29:33.309729+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:29:33.309729+0000","last_undegraded":"2023-03-25T04:29:33.309729+0000","last_fullsized":"2023-03-25T04:29:33.309729+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:23.308607+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:29:23.308607+0000","last_peered":"2023-03-25T04:29:23.308607+0000","last_clean":"2023-03-25T04:29:23.308607+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:29:23.308607+0000","last_undegraded":"2023-03-25T04:29:23.308607+0000","last_fullsized":"2023-03-25T04:29:23.308607+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":28,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:20.554710+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:29:20.554710+0000","last_peered":"2023-03-25T04:29:20.554710+0000","last_clean":"2023-03-25T04:29:20.554710+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:29:20.554710+0000","last_undegraded":"2023-03-25T04:29:20.554710+0000","last_fullsized":"2023-03-25T04:29:20.554710+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309216+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:29:28.309216+0000","last_peered":"2023-03-25T04:29:28.309216+0000","last_clean":"2023-03-25T04:29:28.309216+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:29:28.309216+0000","last_undegraded":"2023-03-25T04:29:28.309216+0000","last_fullsized":"2023-03-25T04:29:28.309216+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309233+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:29:28.309233+0000","last_peered":"2023-03-25T04:29:28.309233+0000","last_clean":"2023-03-25T04:29:28.309233+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:29:28.309233+0000","last_undegraded":"2023-03-25T04:29:28.309233+0000","last_fullsized":"2023-03-25T04:29:28.309233+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117058,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:31:42 2023","interfaces":[{"interface":"back","average":{"1min":0.27500000000000002,"5min":0.28399999999999997,"15min":0.29399999999999998},"min":{"1min":0.19800000000000001,"5min":0.191,"15min":0.191},"max":{"1min":0.36899999999999999,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.30599999999999999},{"interface":"front","average":{"1min":0.27900000000000003,"5min":0.28499999999999998,"15min":0.29599999999999999},"min":{"1min":0.216,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":0.40799999999999997,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.32500000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051649,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:31:35 2023","interfaces":[{"interface":"back","average":{"1min":0.26100000000000001,"5min":0.25700000000000001,"15min":0.251},"min":{"1min":0.19,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.35899999999999999,"5min":0.41599999999999998,"15min":0.41599999999999998},"last":0.24199999999999999},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.27400000000000002,"15min":0.26700000000000002},"min":{"1min":0.20200000000000001,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.374,"5min":0.42899999999999999,"15min":0.42899999999999999},"last":0.22600000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:32:54.944+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:32:55.469+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:32:55.927+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)

Ignoring malparsed backtrace: 2023-03-25T04:33:13.958 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean
, 2023-03-25T04:33:13.959 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}
, 2023-03-25T04:33:13.959 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json

Ignoring malparsed backtrace: 2023-03-25T04:33:31.199 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":267,"stamp":"2023-03-25T04:33:30.166660+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002269"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:33.309748+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:29:33.309748+0000","last_peered":"2023-03-25T04:29:33.309748+0000","last_clean":"2023-03-25T04:29:33.309748+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:29:33.309748+0000","last_undegraded":"2023-03-25T04:29:33.309748+0000","last_fullsized":"2023-03-25T04:29:33.309748+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:33.309729+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:29:33.309729+0000","last_peered":"2023-03-25T04:29:33.309729+0000","last_clean":"2023-03-25T04:29:33.309729+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:29:33.309729+0000","last_undegraded":"2023-03-25T04:29:33.309729+0000","last_fullsized":"2023-03-25T04:29:33.309729+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":28,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:31:23.320894+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:31:23.320894+0000","last_peered":"2023-03-25T04:31:23.320894+0000","last_clean":"2023-03-25T04:31:23.320894+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:31:23.320894+0000","last_undegraded":"2023-03-25T04:31:23.320894+0000","last_fullsized":"2023-03-25T04:31:23.320894+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":28,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:20.554710+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:29:20.554710+0000","last_peered":"2023-03-25T04:29:20.554710+0000","last_clean":"2023-03-25T04:29:20.554710+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:29:20.554710+0000","last_undegraded":"2023-03-25T04:29:20.554710+0000","last_fullsized":"2023-03-25T04:29:20.554710+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309216+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:29:28.309216+0000","last_peered":"2023-03-25T04:29:28.309216+0000","last_clean":"2023-03-25T04:29:28.309216+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:29:28.309216+0000","last_undegraded":"2023-03-25T04:29:28.309216+0000","last_fullsized":"2023-03-25T04:29:28.309216+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309233+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:29:28.309233+0000","last_peered":"2023-03-25T04:29:28.309233+0000","last_clean":"2023-03-25T04:29:28.309233+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:29:28.309233+0000","last_undegraded":"2023-03-25T04:29:28.309233+0000","last_fullsized":"2023-03-25T04:29:28.309233+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117068,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:32:46 2023","interfaces":[{"interface":"back","average":{"1min":0.27000000000000002,"5min":0.27900000000000003,"15min":0.29199999999999998},"min":{"1min":0.17899999999999999,"5min":0.17899999999999999,"15min":0.17899999999999999},"max":{"1min":0.35599999999999998,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.28799999999999998},{"interface":"front","average":{"1min":0.26800000000000002,"5min":0.27800000000000002,"15min":0.29399999999999998},"min":{"1min":0.19900000000000001,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":0.377,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.315}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051660,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:32:36 2023","interfaces":[{"interface":"back","average":{"1min":0.248,"5min":0.25700000000000001,"15min":0.251},"min":{"1min":0.17000000000000001,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"last":0.22700000000000001},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.27600000000000002,"15min":0.26800000000000002},"min":{"1min":0.192,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"last":0.20399999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:33:31.201 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:33:31.400+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)

Ignoring malparsed backtrace: 2023-03-25T04:33:47.531+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:33:47.951+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:33:48.505+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)

Ignoring malparsed backtrace: 2023-03-25T04:34:06.104 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json
, 2023-03-25T04:34:06.118+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:34:06.497 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:34:23.399 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:34:23.665+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:34:23.712 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:34:40.195+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:34:40.523+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:34:40.676 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace: 2023-03-25T04:35:03.381+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:35:04.139+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:35:04.391+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)

Ignoring malparsed backtrace: 2023-03-25T04:35:22.738 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean
, 2023-03-25T04:35:22.739 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}
, 2023-03-25T04:35:22.739 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json

Ignoring malparsed backtrace: 2023-03-25T04:35:41.393+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:35:42.352+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:35:42.376+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:35:57.384 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":340,"stamp":"2023-03-25T04:35:56.194992+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.001930"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":36,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:31:43.322894+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:31:43.322894+0000","last_peered":"2023-03-25T04:31:43.322894+0000","last_clean":"2023-03-25T04:31:43.322894+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:31:43.322894+0000","last_undegraded":"2023-03-25T04:31:43.322894+0000","last_fullsized":"2023-03-25T04:31:43.322894+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":28,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:31:43.322876+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:31:43.322876+0000","last_peered":"2023-03-25T04:31:43.322876+0000","last_clean":"2023-03-25T04:31:43.322876+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:31:43.322876+0000","last_undegraded":"2023-03-25T04:31:43.322876+0000","last_fullsized":"2023-03-25T04:31:43.322876+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:23.331998+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:33:23.331998+0000","last_peered":"2023-03-25T04:33:23.331998+0000","last_clean":"2023-03-25T04:33:23.331998+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:33:23.331998+0000","last_undegraded":"2023-03-25T04:33:23.331998+0000","last_fullsized":"2023-03-25T04:33:23.331998+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:30.576723+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:33:30.576723+0000","last_peered":"2023-03-25T04:33:30.576723+0000","last_clean":"2023-03-25T04:33:30.576723+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:33:30.576723+0000","last_undegraded":"2023-03-25T04:33:30.576723+0000","last_fullsized":"2023-03-25T04:33:30.576723+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":37,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:38.333447+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:33:38.333447+0000","last_peered":"2023-03-25T04:33:38.333447+0000","last_clean":"2023-03-25T04:33:38.333447+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:33:38.333447+0000","last_undegraded":"2023-03-25T04:33:38.333447+0000","last_fullsized":"2023-03-25T04:33:38.333447+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:38.333467+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:33:38.333467+0000","last_peered":"2023-03-25T04:33:38.333467+0000","last_clean":"2023-03-25T04:33:38.333467+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:33:38.333467+0000","last_undegraded":"2023-03-25T04:33:38.333467+0000","last_fullsized":"2023-03-25T04:33:38.333467+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117098,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:35:49 2023","interfaces":[{"interface":"back","average":{"1min":0.255,"5min":0.26000000000000001,"15min":0.28199999999999997},"min":{"1min":0.17899999999999999,"5min":0.16,"15min":0.16},"max":{"1min":0.36399999999999999,"5min":0.42599999999999999,"15min":0.45100000000000001},"last":0.26100000000000001},{"interface":"front","average":{"1min":0.26000000000000001,"5min":0.26700000000000002,"15min":0.28599999999999998},"min":{"1min":0.188,"5min":0.17999999999999999,"15min":0.17499999999999999},"max":{"1min":0.39100000000000001,"5min":0.45200000000000001,"15min":0.46300000000000002},"last":0.28399999999999997}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051689,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:35:42 2023","interfaces":[{"interface":"back","average":{"1min":0.29399999999999998,"5min":0.27200000000000002,"15min":0.25800000000000001},"min":{"1min":0.17299999999999999,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.41099999999999998,"5min":0.42699999999999999,"15min":0.42699999999999999},"last":0.20399999999999999},{"interface":"front","average":{"1min":0.30299999999999999,"5min":0.28399999999999997,"15min":0.27300000000000002},"min":{"1min":0.19700000000000001,"5min":0.192,"15min":0.16600000000000001},"max":{"1min":0.43099999999999999,"5min":0.45800000000000002,"15min":0.45800000000000002},"last":0.22}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:35:57.624+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)

Ignoring malparsed backtrace: 2023-03-25T04:36:17.226 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:36:17.239 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":350,"stamp":"2023-03-25T04:36:16.198866+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002148"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":37,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:53.335026+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:33:53.335026+0000","last_peered":"2023-03-25T04:33:53.335026+0000","last_clean":"2023-03-25T04:33:53.335026+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:33:53.335026+0000","last_undegraded":"2023-03-25T04:33:53.335026+0000","last_fullsized":"2023-03-25T04:33:53.335026+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:53.335009+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:33:53.335009+0000","last_peered":"2023-03-25T04:33:53.335009+0000","last_clean":"2023-03-25T04:33:53.335009+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:33:53.335009+0000","last_undegraded":"2023-03-25T04:33:53.335009+0000","last_fullsized":"2023-03-25T04:33:53.335009+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:23.331998+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:33:23.331998+0000","last_peered":"2023-03-25T04:33:23.331998+0000","last_clean":"2023-03-25T04:33:23.331998+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:33:23.331998+0000","last_undegraded":"2023-03-25T04:33:23.331998+0000","last_fullsized":"2023-03-25T04:33:23.331998+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:30.576723+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:33:30.576723+0000","last_peered":"2023-03-25T04:33:30.576723+0000","last_clean":"2023-03-25T04:33:30.576723+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:33:30.576723+0000","last_undegraded":"2023-03-25T04:33:30.576723+0000","last_fullsized":"2023-03-25T04:33:30.576723+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":37,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:38.333447+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:33:38.333447+0000","last_peered":"2023-03-25T04:33:38.333447+0000","last_clean":"2023-03-25T04:33:38.333447+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:33:38.333447+0000","last_undegraded":"2023-03-25T04:33:38.333447+0000","last_fullsized":"2023-03-25T04:33:38.333447+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:38.333467+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:33:38.333467+0000","last_peered":"2023-03-25T04:33:38.333467+0000","last_clean":"2023-03-25T04:33:38.333467+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:33:38.333467+0000","last_undegraded":"2023-03-25T04:33:38.333467+0000","last_fullsized":"2023-03-25T04:33:38.333467+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117102,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:35:49 2023","interfaces":[{"interface":"back","average":{"1min":0.255,"5min":0.26000000000000001,"15min":0.28199999999999997},"min":{"1min":0.17899999999999999,"5min":0.16,"15min":0.16},"max":{"1min":0.36399999999999999,"5min":0.42599999999999999,"15min":0.45100000000000001},"last":0.28399999999999997},{"interface":"front","average":{"1min":0.26000000000000001,"5min":0.26700000000000002,"15min":0.28599999999999998},"min":{"1min":0.188,"5min":0.17999999999999999,"15min":0.17499999999999999},"max":{"1min":0.39100000000000001,"5min":0.45200000000000001,"15min":0.46300000000000002},"last":0.30299999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051693,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:35:42 2023","interfaces":[{"interface":"back","average":{"1min":0.29399999999999998,"5min":0.27200000000000002,"15min":0.25800000000000001},"min":{"1min":0.17299999999999999,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.41099999999999998,"5min":0.42699999999999999,"15min":0.42699999999999999},"last":0.35099999999999998},{"interface":"front","average":{"1min":0.30299999999999999,"5min":0.28399999999999997,"15min":0.27300000000000002},"min":{"1min":0.19700000000000001,"5min":0.192,"15min":0.16600000000000001},"max":{"1min":0.43099999999999999,"5min":0.45800000000000002,"15min":0.45800000000000002},"last":0.32900000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:36:34.556+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:36:34.717+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:36:35.583+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)

Ignoring malparsed backtrace: 2023-03-25T04:36:52.680 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json
, 2023-03-25T04:36:52.701+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:36:53.067 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:37:09.963 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:37:10.273 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all

Ignoring malparsed backtrace: 2023-03-25T04:37:26.856+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:37:27.288 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:37:27.538+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:37:41.903 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":392,"stamp":"2023-03-25T04:37:40.213373+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002012"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":37,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:53.335026+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:33:53.335026+0000","last_peered":"2023-03-25T04:33:53.335026+0000","last_clean":"2023-03-25T04:33:53.335026+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:33:53.335026+0000","last_undegraded":"2023-03-25T04:33:53.335026+0000","last_fullsized":"2023-03-25T04:33:53.335026+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:53.335009+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:33:53.335009+0000","last_peered":"2023-03-25T04:33:53.335009+0000","last_clean":"2023-03-25T04:33:53.335009+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:33:53.335009+0000","last_undegraded":"2023-03-25T04:33:53.335009+0000","last_fullsized":"2023-03-25T04:33:53.335009+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:23.343585+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:35:23.343585+0000","last_peered":"2023-03-25T04:35:23.343585+0000","last_clean":"2023-03-25T04:35:23.343585+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:35:23.343585+0000","last_undegraded":"2023-03-25T04:35:23.343585+0000","last_fullsized":"2023-03-25T04:35:23.343585+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:30.576723+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:33:30.576723+0000","last_peered":"2023-03-25T04:33:30.576723+0000","last_clean":"2023-03-25T04:33:30.576723+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:33:30.576723+0000","last_undegraded":"2023-03-25T04:33:30.576723+0000","last_fullsized":"2023-03-25T04:33:30.576723+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":37,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:38.333447+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:33:38.333447+0000","last_peered":"2023-03-25T04:33:38.333447+0000","last_clean":"2023-03-25T04:33:38.333447+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:33:38.333447+0000","last_undegraded":"2023-03-25T04:33:38.333447+0000","last_fullsized":"2023-03-25T04:33:38.333447+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:38.333467+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:33:38.333467+0000","last_peered":"2023-03-25T04:33:38.333467+0000","last_clean":"2023-03-25T04:33:38.333467+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:33:38.333467+0000","last_undegraded":"2023-03-25T04:33:38.333467+0000","last_fullsized":"2023-03-25T04:33:38.333467+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117119,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:36:50 2023","interfaces":[{"interface":"back","average":{"1min":0.254,"5min":0.25600000000000001,"15min":0.27900000000000003},"min":{"1min":0.20499999999999999,"5min":0.16,"15min":0.16},"max":{"1min":0.371,"5min":0.42599999999999999,"15min":0.45100000000000001},"last":0.23000000000000001},{"interface":"front","average":{"1min":0.26800000000000002,"5min":0.26500000000000001,"15min":0.28399999999999997},"min":{"1min":0.19700000000000001,"5min":0.17999999999999999,"15min":0.17499999999999999},"max":{"1min":0.34699999999999998,"5min":0.45200000000000001,"15min":0.46300000000000002},"last":0.20899999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051710,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:36:44 2023","interfaces":[{"interface":"back","average":{"1min":0.28399999999999997,"5min":0.27700000000000002,"15min":0.26100000000000001},"min":{"1min":0.192,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.40899999999999997,"5min":0.42699999999999999,"15min":0.42699999999999999},"last":0.39700000000000002},{"interface":"front","average":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.27500000000000002},"min":{"1min":0.182,"5min":0.182,"15min":0.16600000000000001},"max":{"1min":0.439,"5min":0.45800000000000002,"15min":0.45800000000000002},"last":0.439}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:37:42.400+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)

Ignoring malparsed backtrace: 2023-03-25T04:38:04.766+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:38:04.922 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:38:05.246 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:38:19.605 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":411,"stamp":"2023-03-25T04:38:18.220966+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002368"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":38,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:36:03.347070+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:36:03.347070+0000","last_peered":"2023-03-25T04:36:03.347070+0000","last_clean":"2023-03-25T04:36:03.347070+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:36:03.347070+0000","last_undegraded":"2023-03-25T04:36:03.347070+0000","last_fullsized":"2023-03-25T04:36:03.347070+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:36:03.347025+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:36:03.347025+0000","last_peered":"2023-03-25T04:36:03.347025+0000","last_clean":"2023-03-25T04:36:03.347025+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:36:03.347025+0000","last_undegraded":"2023-03-25T04:36:03.347025+0000","last_fullsized":"2023-03-25T04:36:03.347025+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:23.343585+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:35:23.343585+0000","last_peered":"2023-03-25T04:35:23.343585+0000","last_clean":"2023-03-25T04:35:23.343585+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:35:23.343585+0000","last_undegraded":"2023-03-25T04:35:23.343585+0000","last_fullsized":"2023-03-25T04:35:23.343585+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:35.587025+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:35:35.587025+0000","last_peered":"2023-03-25T04:35:35.587025+0000","last_clean":"2023-03-25T04:35:35.587025+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:35:35.587025+0000","last_undegraded":"2023-03-25T04:35:35.587025+0000","last_fullsized":"2023-03-25T04:35:35.587025+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":38,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:43.345308+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:35:43.345308+0000","last_peered":"2023-03-25T04:35:43.345308+0000","last_clean":"2023-03-25T04:35:43.345308+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:35:43.345308+0000","last_undegraded":"2023-03-25T04:35:43.345308+0000","last_fullsized":"2023-03-25T04:35:43.345308+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:43.345324+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:35:43.345324+0000","last_peered":"2023-03-25T04:35:43.345324+0000","last_clean":"2023-03-25T04:35:43.345324+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:35:43.345324+0000","last_undegraded":"2023-03-25T04:35:43.345324+0000","last_fullsized":"2023-03-25T04:35:43.345324+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117127,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:37:53 2023","interfaces":[{"interface":"back","average":{"1min":0.25800000000000001,"5min":0.253,"15min":0.27700000000000002},"min":{"1min":0.184,"5min":0.16,"15min":0.16},"max":{"1min":0.44600000000000001,"5min":0.44600000000000001,"15min":0.45100000000000001},"last":0.35199999999999998},{"interface":"front","average":{"1min":0.26900000000000002,"5min":0.26500000000000001,"15min":0.28199999999999997},"min":{"1min":0.192,"5min":0.17999999999999999,"15min":0.17499999999999999},"max":{"1min":0.46400000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"last":0.33200000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051717,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:37:44 2023","interfaces":[{"interface":"back","average":{"1min":0.27300000000000002,"5min":0.28199999999999997,"15min":0.26200000000000001},"min":{"1min":0.17899999999999999,"5min":0.17299999999999999,"15min":0.17000000000000001},"max":{"1min":0.41799999999999998,"5min":0.42099999999999999,"15min":0.42699999999999999},"last":0.28699999999999998},{"interface":"front","average":{"1min":0.29399999999999998,"5min":0.29099999999999998,"15min":0.27700000000000002},"min":{"1min":0.19900000000000001,"5min":0.182,"15min":0.16600000000000001},"max":{"1min":0.441,"5min":0.441,"15min":0.45800000000000002},"last":0.26300000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:38:19.718+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:38:20.315+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)

Ignoring malparsed backtrace: 2023-03-25T04:38:44.283 INFO:teuthology.orchestra.run.smithi070.stdout:
, 2023-03-25T04:38:44.283 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":781,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":16,"modified":"2023-03-25T04:38:40.225321+0000","services":{}},"progress_events":{}}
, 2023-03-25T04:38:44.296 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now

Ignoring malparsed backtrace: 2023-03-25T04:39:02.477+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:39:02.578+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:39:03.524+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:39:20.359 INFO:teuthology.orchestra.run.smithi070.stdout:
, 2023-03-25T04:39:20.359 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":818,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":16,"modified":"2023-03-25T04:38:40.225321+0000","services":{}},"progress_events":{}}
, 2023-03-25T04:39:20.372 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now

Ignoring malparsed backtrace: 2023-03-25T04:39:37.527 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:39:37.541 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":450,"stamp":"2023-03-25T04:39:36.235332+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002086"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":38,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:36:03.347070+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:36:03.347070+0000","last_peered":"2023-03-25T04:36:03.347070+0000","last_clean":"2023-03-25T04:36:03.347070+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:36:03.347070+0000","last_undegraded":"2023-03-25T04:36:03.347070+0000","last_fullsized":"2023-03-25T04:36:03.347070+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:36:03.347025+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:36:03.347025+0000","last_peered":"2023-03-25T04:36:03.347025+0000","last_clean":"2023-03-25T04:36:03.347025+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:36:03.347025+0000","last_undegraded":"2023-03-25T04:36:03.347025+0000","last_fullsized":"2023-03-25T04:36:03.347025+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:23.353778+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:37:23.353778+0000","last_peered":"2023-03-25T04:37:23.353778+0000","last_clean":"2023-03-25T04:37:23.353778+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:37:23.353778+0000","last_undegraded":"2023-03-25T04:37:23.353778+0000","last_fullsized":"2023-03-25T04:37:23.353778+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:35.587025+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:35:35.587025+0000","last_peered":"2023-03-25T04:35:35.587025+0000","last_clean":"2023-03-25T04:35:35.587025+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:35:35.587025+0000","last_undegraded":"2023-03-25T04:35:35.587025+0000","last_fullsized":"2023-03-25T04:35:35.587025+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":38,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:43.345308+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:35:43.345308+0000","last_peered":"2023-03-25T04:35:43.345308+0000","last_clean":"2023-03-25T04:35:43.345308+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:35:43.345308+0000","last_undegraded":"2023-03-25T04:35:43.345308+0000","last_fullsized":"2023-03-25T04:35:43.345308+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:43.345324+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:35:43.345324+0000","last_peered":"2023-03-25T04:35:43.345324+0000","last_clean":"2023-03-25T04:35:43.345324+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:35:43.345324+0000","last_undegraded":"2023-03-25T04:35:43.345324+0000","last_fullsized":"2023-03-25T04:35:43.345324+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117143,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:38:54 2023","interfaces":[{"interface":"back","average":{"1min":0.25800000000000001,"5min":0.25700000000000001,"15min":0.27400000000000002},"min":{"1min":0.192,"5min":0.17399999999999999,"15min":0.16},"max":{"1min":0.39400000000000002,"5min":0.44600000000000001,"15min":0.45100000000000001},"last":0.26500000000000001},{"interface":"front","average":{"1min":0.25900000000000001,"5min":0.26700000000000002,"15min":0.27900000000000003},"min":{"1min":0.19500000000000001,"5min":0.188,"15min":0.17499999999999999},"max":{"1min":0.41499999999999998,"5min":0.46400000000000002,"15min":0.46400000000000002},"last":0.29099999999999998}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051733,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:38:45 2023","interfaces":[{"interface":"back","average":{"1min":0.27300000000000002,"5min":0.28100000000000003,"15min":0.26400000000000001},"min":{"1min":0.19400000000000001,"5min":0.17299999999999999,"15min":0.17000000000000001},"max":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"last":0.27300000000000002},{"interface":"front","average":{"1min":0.28799999999999998,"5min":0.29099999999999998,"15min":0.27800000000000002},"min":{"1min":0.19800000000000001,"5min":0.182,"15min":0.16600000000000001},"max":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"last":0.308}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:39:54.416+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:39:54.455 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:39:54.780 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:40:09.288+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:40:09.389+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:40:10.267+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)

Ignoring malparsed backtrace: 2023-03-25T04:40:28.984 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":476,"stamp":"2023-03-25T04:40:28.245911+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002693"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":39,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:38:13.358968+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:38:13.358968+0000","last_peered":"2023-03-25T04:38:13.358968+0000","last_clean":"2023-03-25T04:38:13.358968+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:38:13.358968+0000","last_undegraded":"2023-03-25T04:38:13.358968+0000","last_fullsized":"2023-03-25T04:38:13.358968+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:38:13.358922+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:38:13.358922+0000","last_peered":"2023-03-25T04:38:13.358922+0000","last_clean":"2023-03-25T04:38:13.358922+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:38:13.358922+0000","last_undegraded":"2023-03-25T04:38:13.358922+0000","last_fullsized":"2023-03-25T04:38:13.358922+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:23.353778+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:37:23.353778+0000","last_peered":"2023-03-25T04:37:23.353778+0000","last_clean":"2023-03-25T04:37:23.353778+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:37:23.353778+0000","last_undegraded":"2023-03-25T04:37:23.353778+0000","last_fullsized":"2023-03-25T04:37:23.353778+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:40.598579+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:37:40.598579+0000","last_peered":"2023-03-25T04:37:40.598579+0000","last_clean":"2023-03-25T04:37:40.598579+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:37:40.598579+0000","last_undegraded":"2023-03-25T04:37:40.598579+0000","last_fullsized":"2023-03-25T04:37:40.598579+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":39,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:48.356344+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:37:48.356344+0000","last_peered":"2023-03-25T04:37:48.356344+0000","last_clean":"2023-03-25T04:37:48.356344+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:37:48.356344+0000","last_undegraded":"2023-03-25T04:37:48.356344+0000","last_fullsized":"2023-03-25T04:37:48.356344+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:48.356365+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:37:48.356365+0000","last_peered":"2023-03-25T04:37:48.356365+0000","last_clean":"2023-03-25T04:37:48.356365+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:37:48.356365+0000","last_undegraded":"2023-03-25T04:37:48.356365+0000","last_fullsized":"2023-03-25T04:37:48.356365+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117153,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:39:54 2023","interfaces":[{"interface":"back","average":{"1min":0.25900000000000001,"5min":0.25600000000000001,"15min":0.27100000000000002},"min":{"1min":0.188,"5min":0.17899999999999999,"15min":0.16},"max":{"1min":0.41999999999999998,"5min":0.44600000000000001,"15min":0.45100000000000001},"last":0.255},{"interface":"front","average":{"1min":0.27500000000000002,"5min":0.26600000000000001,"15min":0.27700000000000002},"min":{"1min":0.17699999999999999,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.45600000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"last":0.26800000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051743,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:39:47 2023","interfaces":[{"interface":"back","average":{"1min":0.26300000000000001,"5min":0.27700000000000002,"15min":0.26500000000000001},"min":{"1min":0.187,"5min":0.17299999999999999,"15min":0.17000000000000001},"max":{"1min":0.433,"5min":0.433,"15min":0.433},"last":0.36599999999999999},{"interface":"front","average":{"1min":0.27100000000000002,"5min":0.28799999999999998,"15min":0.27900000000000003},"min":{"1min":0.20100000000000001,"5min":0.182,"15min":0.16600000000000001},"max":{"1min":0.48599999999999999,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.34999999999999998}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:40:28.986 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean
, 2023-03-25T04:40:28.986 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}

Ignoring malparsed backtrace: 2023-03-25T04:40:47.229+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:40:47.503+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:40:48.251+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)

Ignoring malparsed backtrace: 2023-03-25T04:41:03.267 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:41:03.578 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all

Ignoring malparsed backtrace: 2023-03-25T04:41:21.347 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":502,"stamp":"2023-03-25T04:41:20.256204+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002342"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":39,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:38:13.358968+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:38:13.358968+0000","last_peered":"2023-03-25T04:38:13.358968+0000","last_clean":"2023-03-25T04:38:13.358968+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:38:13.358968+0000","last_undegraded":"2023-03-25T04:38:13.358968+0000","last_fullsized":"2023-03-25T04:38:13.358968+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:38:13.358922+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:38:13.358922+0000","last_peered":"2023-03-25T04:38:13.358922+0000","last_clean":"2023-03-25T04:38:13.358922+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:38:13.358922+0000","last_undegraded":"2023-03-25T04:38:13.358922+0000","last_fullsized":"2023-03-25T04:38:13.358922+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:23.353778+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:37:23.353778+0000","last_peered":"2023-03-25T04:37:23.353778+0000","last_clean":"2023-03-25T04:37:23.353778+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:37:23.353778+0000","last_undegraded":"2023-03-25T04:37:23.353778+0000","last_fullsized":"2023-03-25T04:37:23.353778+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:40.598579+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:37:40.598579+0000","last_peered":"2023-03-25T04:37:40.598579+0000","last_clean":"2023-03-25T04:37:40.598579+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:37:40.598579+0000","last_undegraded":"2023-03-25T04:37:40.598579+0000","last_fullsized":"2023-03-25T04:37:40.598579+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":39,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:48.356344+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:37:48.356344+0000","last_peered":"2023-03-25T04:37:48.356344+0000","last_clean":"2023-03-25T04:37:48.356344+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:37:48.356344+0000","last_undegraded":"2023-03-25T04:37:48.356344+0000","last_fullsized":"2023-03-25T04:37:48.356344+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:48.356365+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:37:48.356365+0000","last_peered":"2023-03-25T04:37:48.356365+0000","last_clean":"2023-03-25T04:37:48.356365+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:37:48.356365+0000","last_undegraded":"2023-03-25T04:37:48.356365+0000","last_fullsized":"2023-03-25T04:37:48.356365+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117163,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:40:56 2023","interfaces":[{"interface":"back","average":{"1min":0.28799999999999998,"5min":0.26300000000000001,"15min":0.27100000000000002},"min":{"1min":0.188,"5min":0.184,"15min":0.16},"max":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.17499999999999999},{"interface":"front","average":{"1min":0.29599999999999999,"5min":0.27300000000000002,"15min":0.27700000000000002},"min":{"1min":0.20599999999999999,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.43099999999999999,"5min":0.46400000000000002,"15min":0.46400000000000002},"last":0.20699999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051754,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:40:52 2023","interfaces":[{"interface":"back","average":{"1min":0.28599999999999998,"5min":0.27500000000000002,"15min":0.26700000000000002},"min":{"1min":0.17899999999999999,"5min":0.17899999999999999,"15min":0.17000000000000001},"max":{"1min":0.44,"5min":0.44,"15min":0.44},"last":0.23699999999999999},{"interface":"front","average":{"1min":0.29799999999999999,"5min":0.28699999999999998,"15min":0.28100000000000003},"min":{"1min":0.20300000000000001,"5min":0.182,"15min":0.16600000000000001},"max":{"1min":0.47599999999999998,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.28000000000000003}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:41:21.349 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean
, 2023-03-25T04:41:21.349 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}

Ignoring malparsed backtrace: 2023-03-25T04:41:39.867+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:41:40.383+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:41:40.905+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:41:58.219 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json
, 2023-03-25T04:41:58.319+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:41:58.644 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:42:18.193 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":530,"stamp":"2023-03-25T04:42:16.266659+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002356"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":39,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:38:13.358968+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:38:13.358968+0000","last_peered":"2023-03-25T04:38:13.358968+0000","last_clean":"2023-03-25T04:38:13.358968+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:38:13.358968+0000","last_undegraded":"2023-03-25T04:38:13.358968+0000","last_fullsized":"2023-03-25T04:38:13.358968+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:38:13.358922+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:38:13.358922+0000","last_peered":"2023-03-25T04:38:13.358922+0000","last_clean":"2023-03-25T04:38:13.358922+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:38:13.358922+0000","last_undegraded":"2023-03-25T04:38:13.358922+0000","last_fullsized":"2023-03-25T04:38:13.358922+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:23.364593+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:39:23.364593+0000","last_peered":"2023-03-25T04:39:23.364593+0000","last_clean":"2023-03-25T04:39:23.364593+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:39:23.364593+0000","last_undegraded":"2023-03-25T04:39:23.364593+0000","last_fullsized":"2023-03-25T04:39:23.364593+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:45.608475+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:39:45.608475+0000","last_peered":"2023-03-25T04:39:45.608475+0000","last_clean":"2023-03-25T04:39:45.608475+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:39:45.608475+0000","last_undegraded":"2023-03-25T04:39:45.608475+0000","last_fullsized":"2023-03-25T04:39:45.608475+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":40,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:53.367398+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:39:53.367398+0000","last_peered":"2023-03-25T04:39:53.367398+0000","last_clean":"2023-03-25T04:39:53.367398+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:39:53.367398+0000","last_undegraded":"2023-03-25T04:39:53.367398+0000","last_fullsized":"2023-03-25T04:39:53.367398+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:53.367424+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:39:53.367424+0000","last_peered":"2023-03-25T04:39:53.367424+0000","last_clean":"2023-03-25T04:39:53.367424+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:39:53.367424+0000","last_undegraded":"2023-03-25T04:39:53.367424+0000","last_fullsized":"2023-03-25T04:39:53.367424+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117175,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:41:58 2023","interfaces":[{"interface":"back","average":{"1min":0.25800000000000001,"5min":0.26400000000000001,"15min":0.26800000000000002},"min":{"1min":0.17499999999999999,"5min":0.17499999999999999,"15min":0.16},"max":{"1min":0.39100000000000001,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.22900000000000001},{"interface":"front","average":{"1min":0.26300000000000001,"5min":0.27200000000000002,"15min":0.27400000000000002},"min":{"1min":0.20599999999999999,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.35699999999999998,"5min":0.46400000000000002,"15min":0.46400000000000002},"last":0.26400000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051765,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:41:52 2023","interfaces":[{"interface":"back","average":{"1min":0.26500000000000001,"5min":0.27200000000000002,"15min":0.26800000000000002},"min":{"1min":0.17000000000000001,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.39200000000000002,"5min":0.44,"15min":0.44},"last":0.23899999999999999},{"interface":"front","average":{"1min":0.28100000000000003,"5min":0.28599999999999998,"15min":0.28199999999999997},"min":{"1min":0.19400000000000001,"5min":0.19400000000000001,"15min":0.16600000000000001},"max":{"1min":0.41399999999999998,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.254}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:42:18.297+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:42:18.958+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:42:35.512 INFO:teuthology.orchestra.run.smithi070.stdout:
, 2023-03-25T04:42:35.512 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":1013,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":17,"modified":"2023-03-25T04:41:54.262744+0000","services":{}},"progress_events":{}}
, 2023-03-25T04:42:35.525 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:42:52.741 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":548,"stamp":"2023-03-25T04:42:52.273601+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002221"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":40,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:40:23.370056+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:40:23.370056+0000","last_peered":"2023-03-25T04:40:23.370056+0000","last_clean":"2023-03-25T04:40:23.370056+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:40:23.370056+0000","last_undegraded":"2023-03-25T04:40:23.370056+0000","last_fullsized":"2023-03-25T04:40:23.370056+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:40:23.370039+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:40:23.370039+0000","last_peered":"2023-03-25T04:40:23.370039+0000","last_clean":"2023-03-25T04:40:23.370039+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:40:23.370039+0000","last_undegraded":"2023-03-25T04:40:23.370039+0000","last_fullsized":"2023-03-25T04:40:23.370039+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:23.364593+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:39:23.364593+0000","last_peered":"2023-03-25T04:39:23.364593+0000","last_clean":"2023-03-25T04:39:23.364593+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:39:23.364593+0000","last_undegraded":"2023-03-25T04:39:23.364593+0000","last_fullsized":"2023-03-25T04:39:23.364593+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:45.608475+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:39:45.608475+0000","last_peered":"2023-03-25T04:39:45.608475+0000","last_clean":"2023-03-25T04:39:45.608475+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:39:45.608475+0000","last_undegraded":"2023-03-25T04:39:45.608475+0000","last_fullsized":"2023-03-25T04:39:45.608475+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":40,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:53.367398+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:39:53.367398+0000","last_peered":"2023-03-25T04:39:53.367398+0000","last_clean":"2023-03-25T04:39:53.367398+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:39:53.367398+0000","last_undegraded":"2023-03-25T04:39:53.367398+0000","last_fullsized":"2023-03-25T04:39:53.367398+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:53.367424+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:39:53.367424+0000","last_peered":"2023-03-25T04:39:53.367424+0000","last_clean":"2023-03-25T04:39:53.367424+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:39:53.367424+0000","last_undegraded":"2023-03-25T04:39:53.367424+0000","last_fullsized":"2023-03-25T04:39:53.367424+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117182,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:41:58 2023","interfaces":[{"interface":"back","average":{"1min":0.25800000000000001,"5min":0.26400000000000001,"15min":0.26800000000000002},"min":{"1min":0.17499999999999999,"5min":0.17499999999999999,"15min":0.16},"max":{"1min":0.39100000000000001,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.19600000000000001},{"interface":"front","average":{"1min":0.26300000000000001,"5min":0.27200000000000002,"15min":0.27400000000000002},"min":{"1min":0.20599999999999999,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.35699999999999998,"5min":0.46400000000000002,"15min":0.46400000000000002},"last":0.219}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051773,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:41:52 2023","interfaces":[{"interface":"back","average":{"1min":0.26500000000000001,"5min":0.27200000000000002,"15min":0.26800000000000002},"min":{"1min":0.17000000000000001,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.39200000000000002,"5min":0.44,"15min":0.44},"last":0.19},{"interface":"front","average":{"1min":0.28100000000000003,"5min":0.28599999999999998,"15min":0.28199999999999997},"min":{"1min":0.19400000000000001,"5min":0.19400000000000001,"15min":0.16600000000000001},"max":{"1min":0.41399999999999998,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.17599999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:42:52.742 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean

Ignoring malparsed backtrace: 2023-03-25T04:43:09.973 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:43:09.986 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":556,"stamp":"2023-03-25T04:43:08.276556+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002359"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":40,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:40:23.370056+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:40:23.370056+0000","last_peered":"2023-03-25T04:40:23.370056+0000","last_clean":"2023-03-25T04:40:23.370056+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:40:23.370056+0000","last_undegraded":"2023-03-25T04:40:23.370056+0000","last_fullsized":"2023-03-25T04:40:23.370056+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:40:23.370039+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:40:23.370039+0000","last_peered":"2023-03-25T04:40:23.370039+0000","last_clean":"2023-03-25T04:40:23.370039+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:40:23.370039+0000","last_undegraded":"2023-03-25T04:40:23.370039+0000","last_fullsized":"2023-03-25T04:40:23.370039+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:23.364593+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:39:23.364593+0000","last_peered":"2023-03-25T04:39:23.364593+0000","last_clean":"2023-03-25T04:39:23.364593+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:39:23.364593+0000","last_undegraded":"2023-03-25T04:39:23.364593+0000","last_fullsized":"2023-03-25T04:39:23.364593+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:45.608475+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:39:45.608475+0000","last_peered":"2023-03-25T04:39:45.608475+0000","last_clean":"2023-03-25T04:39:45.608475+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:39:45.608475+0000","last_undegraded":"2023-03-25T04:39:45.608475+0000","last_fullsized":"2023-03-25T04:39:45.608475+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":40,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:53.367398+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:39:53.367398+0000","last_peered":"2023-03-25T04:39:53.367398+0000","last_clean":"2023-03-25T04:39:53.367398+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:39:53.367398+0000","last_undegraded":"2023-03-25T04:39:53.367398+0000","last_fullsized":"2023-03-25T04:39:53.367398+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:53.367424+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:39:53.367424+0000","last_peered":"2023-03-25T04:39:53.367424+0000","last_clean":"2023-03-25T04:39:53.367424+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:39:53.367424+0000","last_undegraded":"2023-03-25T04:39:53.367424+0000","last_fullsized":"2023-03-25T04:39:53.367424+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117185,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:43:00 2023","interfaces":[{"interface":"back","average":{"1min":0.27400000000000002,"5min":0.26700000000000002,"15min":0.26600000000000001},"min":{"1min":0.19600000000000001,"5min":0.17499999999999999,"15min":0.16},"max":{"1min":0.42999999999999999,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.27800000000000002},{"interface":"front","average":{"1min":0.28199999999999997,"5min":0.27500000000000002,"15min":0.27300000000000002},"min":{"1min":0.185,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.47199999999999998,"5min":0.47199999999999998,"15min":0.47199999999999998},"last":0.247}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051776,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:42:54 2023","interfaces":[{"interface":"back","average":{"1min":0.26300000000000001,"5min":0.27000000000000002,"15min":0.26900000000000002},"min":{"1min":0.17399999999999999,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.438,"5min":0.44,"15min":0.44},"last":0.20100000000000001},{"interface":"front","average":{"1min":0.27400000000000002,"5min":0.28199999999999997,"15min":0.28299999999999997},"min":{"1min":0.17599999999999999,"5min":0.17599999999999999,"15min":0.16600000000000001},"max":{"1min":0.41599999999999998,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.23499999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:43:25.042+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:43:25.411+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:43:26.013+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:43:47.937+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:43:48.534+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:43:48.889+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:44:09.318 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:44:09.331 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":586,"stamp":"2023-03-25T04:44:08.288933+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002697"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":40,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:40:23.370056+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:40:23.370056+0000","last_peered":"2023-03-25T04:40:23.370056+0000","last_clean":"2023-03-25T04:40:23.370056+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:40:23.370056+0000","last_undegraded":"2023-03-25T04:40:23.370056+0000","last_fullsized":"2023-03-25T04:40:23.370056+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:40:23.370039+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:40:23.370039+0000","last_peered":"2023-03-25T04:40:23.370039+0000","last_clean":"2023-03-25T04:40:23.370039+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:40:23.370039+0000","last_undegraded":"2023-03-25T04:40:23.370039+0000","last_fullsized":"2023-03-25T04:40:23.370039+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:23.375106+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:41:23.375106+0000","last_peered":"2023-03-25T04:41:23.375106+0000","last_clean":"2023-03-25T04:41:23.375106+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:41:23.375106+0000","last_undegraded":"2023-03-25T04:41:23.375106+0000","last_fullsized":"2023-03-25T04:41:23.375106+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:50.618930+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:41:50.618930+0000","last_peered":"2023-03-25T04:41:50.618930+0000","last_clean":"2023-03-25T04:41:50.618930+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:41:50.618930+0000","last_undegraded":"2023-03-25T04:41:50.618930+0000","last_fullsized":"2023-03-25T04:41:50.618930+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":41,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:58.377821+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:41:58.377821+0000","last_peered":"2023-03-25T04:41:58.377821+0000","last_clean":"2023-03-25T04:41:58.377821+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:41:58.377821+0000","last_undegraded":"2023-03-25T04:41:58.377821+0000","last_fullsized":"2023-03-25T04:41:58.377821+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:58.377836+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:41:58.377836+0000","last_peered":"2023-03-25T04:41:58.377836+0000","last_clean":"2023-03-25T04:41:58.377836+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:41:58.377836+0000","last_undegraded":"2023-03-25T04:41:58.377836+0000","last_fullsized":"2023-03-25T04:41:58.377836+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117197,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:44:01 2023","interfaces":[{"interface":"back","average":{"1min":0.27200000000000002,"5min":0.27000000000000002,"15min":0.26400000000000001},"min":{"1min":0.20399999999999999,"5min":0.17499999999999999,"15min":0.16},"max":{"1min":0.38400000000000001,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.245},{"interface":"front","average":{"1min":0.27600000000000002,"5min":0.27800000000000002,"15min":0.27100000000000002},"min":{"1min":0.23100000000000001,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.40400000000000003,"5min":0.47199999999999998,"15min":0.47199999999999998},"last":0.27500000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051788,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:43:55 2023","interfaces":[{"interface":"back","average":{"1min":0.26100000000000001,"5min":0.26700000000000002,"15min":0.27000000000000002},"min":{"1min":0.17799999999999999,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.374,"5min":0.44,"15min":0.44},"last":0.30299999999999999},{"interface":"front","average":{"1min":0.28299999999999997,"5min":0.28100000000000003,"15min":0.28399999999999997},"min":{"1min":0.20899999999999999,"5min":0.17599999999999999,"15min":0.16600000000000001},"max":{"1min":0.40100000000000002,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.27800000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:44:28.923 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":1126,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":19,"modified":"2023-03-25T04:43:20.278959+0000","services":{}},"progress_events":{}}
, 2023-03-25T04:44:28.936 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now
, 2023-03-25T04:44:28.937 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace: 2023-03-25T04:44:48.786 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:44:48.800 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":606,"stamp":"2023-03-25T04:44:48.296805+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002182"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":41,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:42:34.382196+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:42:34.382196+0000","last_peered":"2023-03-25T04:42:34.382196+0000","last_clean":"2023-03-25T04:42:34.382196+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:42:34.382196+0000","last_undegraded":"2023-03-25T04:42:34.382196+0000","last_fullsized":"2023-03-25T04:42:34.382196+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:42:29.381744+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:42:29.381744+0000","last_peered":"2023-03-25T04:42:29.381744+0000","last_clean":"2023-03-25T04:42:29.381744+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:42:29.381744+0000","last_undegraded":"2023-03-25T04:42:29.381744+0000","last_fullsized":"2023-03-25T04:42:29.381744+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:23.375106+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:41:23.375106+0000","last_peered":"2023-03-25T04:41:23.375106+0000","last_clean":"2023-03-25T04:41:23.375106+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:41:23.375106+0000","last_undegraded":"2023-03-25T04:41:23.375106+0000","last_fullsized":"2023-03-25T04:41:23.375106+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:50.618930+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:41:50.618930+0000","last_peered":"2023-03-25T04:41:50.618930+0000","last_clean":"2023-03-25T04:41:50.618930+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:41:50.618930+0000","last_undegraded":"2023-03-25T04:41:50.618930+0000","last_fullsized":"2023-03-25T04:41:50.618930+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":41,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:58.377821+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:41:58.377821+0000","last_peered":"2023-03-25T04:41:58.377821+0000","last_clean":"2023-03-25T04:41:58.377821+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:41:58.377821+0000","last_undegraded":"2023-03-25T04:41:58.377821+0000","last_fullsized":"2023-03-25T04:41:58.377821+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:58.377836+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:41:58.377836+0000","last_peered":"2023-03-25T04:41:58.377836+0000","last_clean":"2023-03-25T04:41:58.377836+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:41:58.377836+0000","last_undegraded":"2023-03-25T04:41:58.377836+0000","last_fullsized":"2023-03-25T04:41:58.377836+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117206,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:44:01 2023","interfaces":[{"interface":"back","average":{"1min":0.27200000000000002,"5min":0.27000000000000002,"15min":0.26400000000000001},"min":{"1min":0.20399999999999999,"5min":0.17499999999999999,"15min":0.16},"max":{"1min":0.38400000000000001,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.216},{"interface":"front","average":{"1min":0.27600000000000002,"5min":0.27800000000000002,"15min":0.27100000000000002},"min":{"1min":0.23100000000000001,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.40400000000000003,"5min":0.47199999999999998,"15min":0.47199999999999998},"last":0.23999999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051796,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:43:55 2023","interfaces":[{"interface":"back","average":{"1min":0.26100000000000001,"5min":0.26700000000000002,"15min":0.27000000000000002},"min":{"1min":0.17799999999999999,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.374,"5min":0.44,"15min":0.44},"last":0.25600000000000001},{"interface":"front","average":{"1min":0.28299999999999997,"5min":0.28100000000000003,"15min":0.28399999999999997},"min":{"1min":0.20899999999999999,"5min":0.17599999999999999,"15min":0.16600000000000001},"max":{"1min":0.40100000000000002,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.28000000000000003}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:45:05.762 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:45:06.073 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all

Ignoring malparsed backtrace: 2023-03-25T04:45:25.658 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":624,"stamp":"2023-03-25T04:45:24.304294+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002366"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":41,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:42:34.382196+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:42:34.382196+0000","last_peered":"2023-03-25T04:42:34.382196+0000","last_clean":"2023-03-25T04:42:34.382196+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:42:34.382196+0000","last_undegraded":"2023-03-25T04:42:34.382196+0000","last_fullsized":"2023-03-25T04:42:34.382196+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:42:29.381744+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:42:29.381744+0000","last_peered":"2023-03-25T04:42:29.381744+0000","last_clean":"2023-03-25T04:42:29.381744+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:42:29.381744+0000","last_undegraded":"2023-03-25T04:42:29.381744+0000","last_fullsized":"2023-03-25T04:42:29.381744+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:23.375106+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:41:23.375106+0000","last_peered":"2023-03-25T04:41:23.375106+0000","last_clean":"2023-03-25T04:41:23.375106+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:41:23.375106+0000","last_undegraded":"2023-03-25T04:41:23.375106+0000","last_fullsized":"2023-03-25T04:41:23.375106+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:50.618930+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:41:50.618930+0000","last_peered":"2023-03-25T04:41:50.618930+0000","last_clean":"2023-03-25T04:41:50.618930+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:41:50.618930+0000","last_undegraded":"2023-03-25T04:41:50.618930+0000","last_fullsized":"2023-03-25T04:41:50.618930+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":41,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:58.377821+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:41:58.377821+0000","last_peered":"2023-03-25T04:41:58.377821+0000","last_clean":"2023-03-25T04:41:58.377821+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:41:58.377821+0000","last_undegraded":"2023-03-25T04:41:58.377821+0000","last_fullsized":"2023-03-25T04:41:58.377821+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:58.377836+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:41:58.377836+0000","last_peered":"2023-03-25T04:41:58.377836+0000","last_clean":"2023-03-25T04:41:58.377836+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:41:58.377836+0000","last_undegraded":"2023-03-25T04:41:58.377836+0000","last_fullsized":"2023-03-25T04:41:58.377836+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117213,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:45:01 2023","interfaces":[{"interface":"back","average":{"1min":0.251,"5min":0.26800000000000002,"15min":0.26400000000000001},"min":{"1min":0.183,"5min":0.17499999999999999,"15min":0.16},"max":{"1min":0.41499999999999998,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.26900000000000002},{"interface":"front","average":{"1min":0.28000000000000003,"5min":0.27900000000000003,"15min":0.27100000000000002},"min":{"1min":0.20999999999999999,"5min":0.185,"15min":0.17499999999999999},"max":{"1min":0.44400000000000001,"5min":0.47199999999999998,"15min":0.47199999999999998},"last":0.27900000000000003}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051803,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:44:55 2023","interfaces":[{"interface":"back","average":{"1min":0.252,"5min":0.26500000000000001,"15min":0.26900000000000002},"min":{"1min":0.183,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.39700000000000002,"5min":0.44,"15min":0.44},"last":0.20699999999999999},{"interface":"front","average":{"1min":0.27000000000000002,"5min":0.28100000000000003,"15min":0.28299999999999997},"min":{"1min":0.20300000000000001,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":0.42099999999999999,"5min":0.47599999999999998,"15min":0.48599999999999999},"last":0.23599999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:45:25.659 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:45:25.969 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:45:42.735+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:45:42.744+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:45:42.897 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace: 2023-03-25T04:46:00.183 INFO:teuthology.orchestra.run.smithi070.stdout:
, 2023-03-25T04:46:00.183 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":1217,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":20,"modified":"2023-03-25T04:45:42.307700+0000","services":{"mgr":{"daemons":{"summary":"","x":{"start_epoch":0,"start_stamp":"0.000000","gid":0,"addr":"(unrecognized address family 0)/0","metadata":{},"task_status":{}}}}}},"progress_events":{}}
, 2023-03-25T04:46:00.197 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now

Ignoring malparsed backtrace: 2023-03-25T04:46:19.910+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:46:20.039 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all

Ignoring malparsed backtrace: 2023-03-25T04:46:39.611 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}
, 2023-03-25T04:46:39.611 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json
, 2023-03-25T04:46:39.613+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:46:58.485+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:46:58.764+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:46:59.519+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:47:14.132 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:47:14.146 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":678,"stamp":"2023-03-25T04:47:12.325044+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002571"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":42,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:44.393426+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:44:44.393426+0000","last_peered":"2023-03-25T04:44:44.393426+0000","last_clean":"2023-03-25T04:44:44.393426+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:44:44.393426+0000","last_undegraded":"2023-03-25T04:44:44.393426+0000","last_fullsized":"2023-03-25T04:44:44.393426+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:39.393044+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:44:39.393044+0000","last_peered":"2023-03-25T04:44:39.393044+0000","last_clean":"2023-03-25T04:44:39.393044+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:44:39.393044+0000","last_undegraded":"2023-03-25T04:44:39.393044+0000","last_fullsized":"2023-03-25T04:44:39.393044+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:43:24.386177+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:43:24.386177+0000","last_peered":"2023-03-25T04:43:24.386177+0000","last_clean":"2023-03-25T04:43:24.386177+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:43:24.386177+0000","last_undegraded":"2023-03-25T04:43:24.386177+0000","last_fullsized":"2023-03-25T04:43:24.386177+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:43:55.629798+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:43:55.629798+0000","last_peered":"2023-03-25T04:43:55.629798+0000","last_clean":"2023-03-25T04:43:55.629798+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:43:55.629798+0000","last_undegraded":"2023-03-25T04:43:55.629798+0000","last_fullsized":"2023-03-25T04:43:55.629798+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":42,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:04.389904+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:44:04.389904+0000","last_peered":"2023-03-25T04:44:04.389904+0000","last_clean":"2023-03-25T04:44:04.389904+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:44:04.389904+0000","last_undegraded":"2023-03-25T04:44:04.389904+0000","last_fullsized":"2023-03-25T04:44:04.389904+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:04.389923+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:44:04.389923+0000","last_peered":"2023-03-25T04:44:04.389923+0000","last_clean":"2023-03-25T04:44:04.389923+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:44:04.389923+0000","last_undegraded":"2023-03-25T04:44:04.389923+0000","last_fullsized":"2023-03-25T04:44:04.389923+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117235,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:47:05 2023","interfaces":[{"interface":"back","average":{"1min":0.27800000000000002,"5min":0.27000000000000002,"15min":0.26300000000000001},"min":{"1min":0.20100000000000001,"5min":0.16900000000000001,"15min":0.16},"max":{"1min":0.39200000000000002,"5min":0.42999999999999999,"15min":0.45600000000000002},"last":0.379},{"interface":"front","average":{"1min":0.28399999999999997,"5min":0.28100000000000003,"15min":0.27200000000000002},"min":{"1min":0.17799999999999999,"5min":0.17799999999999999,"15min":0.17699999999999999},"max":{"1min":0.42599999999999999,"5min":0.47199999999999998,"15min":0.47199999999999998},"last":0.39800000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051825,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:47:01 2023","interfaces":[{"interface":"back","average":{"1min":0.27300000000000002,"5min":0.26100000000000001,"15min":0.27000000000000002},"min":{"1min":0.186,"5min":0.17399999999999999,"15min":0.17000000000000001},"max":{"1min":0.40400000000000003,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.23899999999999999},{"interface":"front","average":{"1min":0.28699999999999998,"5min":0.27500000000000002,"15min":0.28299999999999997},"min":{"1min":0.186,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.48599999999999999},"last":0.19600000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:47:31.198 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json
, 2023-03-25T04:47:31.411+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:47:31.789 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:47:49.509 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":696,"stamp":"2023-03-25T04:47:48.332093+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002697"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":42,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:44.393426+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:44:44.393426+0000","last_peered":"2023-03-25T04:44:44.393426+0000","last_clean":"2023-03-25T04:44:44.393426+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:44:44.393426+0000","last_undegraded":"2023-03-25T04:44:44.393426+0000","last_fullsized":"2023-03-25T04:44:44.393426+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:39.393044+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:44:39.393044+0000","last_peered":"2023-03-25T04:44:39.393044+0000","last_clean":"2023-03-25T04:44:39.393044+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:44:39.393044+0000","last_undegraded":"2023-03-25T04:44:39.393044+0000","last_fullsized":"2023-03-25T04:44:39.393044+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:45:24.396717+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:45:24.396717+0000","last_peered":"2023-03-25T04:45:24.396717+0000","last_clean":"2023-03-25T04:45:24.396717+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:45:24.396717+0000","last_undegraded":"2023-03-25T04:45:24.396717+0000","last_fullsized":"2023-03-25T04:45:24.396717+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:43:55.629798+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:43:55.629798+0000","last_peered":"2023-03-25T04:43:55.629798+0000","last_clean":"2023-03-25T04:43:55.629798+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:43:55.629798+0000","last_undegraded":"2023-03-25T04:43:55.629798+0000","last_fullsized":"2023-03-25T04:43:55.629798+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":42,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:04.389904+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:44:04.389904+0000","last_peered":"2023-03-25T04:44:04.389904+0000","last_clean":"2023-03-25T04:44:04.389904+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:44:04.389904+0000","last_undegraded":"2023-03-25T04:44:04.389904+0000","last_fullsized":"2023-03-25T04:44:04.389904+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:04.389923+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:44:04.389923+0000","last_peered":"2023-03-25T04:44:04.389923+0000","last_clean":"2023-03-25T04:44:04.389923+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:44:04.389923+0000","last_undegraded":"2023-03-25T04:44:04.389923+0000","last_fullsized":"2023-03-25T04:44:04.389923+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117242,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:47:05 2023","interfaces":[{"interface":"back","average":{"1min":0.27800000000000002,"5min":0.27000000000000002,"15min":0.26300000000000001},"min":{"1min":0.20100000000000001,"5min":0.16900000000000001,"15min":0.16},"max":{"1min":0.39200000000000002,"5min":0.42999999999999999,"15min":0.45600000000000002},"last":0.40500000000000003},{"interface":"front","average":{"1min":0.28399999999999997,"5min":0.28100000000000003,"15min":0.27200000000000002},"min":{"1min":0.17799999999999999,"5min":0.17799999999999999,"15min":0.17699999999999999},"max":{"1min":0.42599999999999999,"5min":0.47199999999999998,"15min":0.47199999999999998},"last":0.42399999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051834,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:47:01 2023","interfaces":[{"interface":"back","average":{"1min":0.27300000000000002,"5min":0.26100000000000001,"15min":0.27000000000000002},"min":{"1min":0.186,"5min":0.17399999999999999,"15min":0.17000000000000001},"max":{"1min":0.40400000000000003,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.25600000000000001},{"interface":"front","average":{"1min":0.28699999999999998,"5min":0.27500000000000002,"15min":0.28299999999999997},"min":{"1min":0.186,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.48599999999999999},"last":0.22800000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:47:49.510 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace:     ctx.managers[config['cluster']].wait_for_clean()
,   File "/home/teuthworker/src/github.com_ceph_ceph-c_511015c23eef0684a64171b710e432448df2b7c7/qa/tasks/ceph_manager.py", line 2747, in wait_for_clean
,     assert time.time() - start < timeout, \

Ignoring malparsed backtrace:  ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
,  1: /lib64/libpthread.so.0(+0x12cf0) [0x7f1453c2fcf0]
,  2: __pthread_mutex_lock()

Ignoring malparsed backtrace: 2023-03-25T04:28:05.172 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":142,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":5,"modified":"2023-03-25T04:27:12.092098+0000","services":{}},"progress_events":{}}
, 2023-03-25T04:28:05.185 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now
, 2023-03-25T04:28:05.186 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace: 2023-03-25T04:28:28.360 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":116,"stamp":"2023-03-25T04:28:28.108007+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002578"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.314661+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:27:45.314661+0000","last_peered":"2023-03-25T04:27:45.314661+0000","last_clean":"2023-03-25T04:27:45.314661+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:27:45.314661+0000","last_undegraded":"2023-03-25T04:27:45.314661+0000","last_fullsized":"2023-03-25T04:27:45.314661+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":25,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.310745+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:27:44.310745+0000","last_peered":"2023-03-25T04:27:44.310745+0000","last_clean":"2023-03-25T04:27:44.310745+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:27:44.310745+0000","last_undegraded":"2023-03-25T04:27:44.310745+0000","last_fullsized":"2023-03-25T04:27:44.310745+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":25,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.310650+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:27:44.310650+0000","last_peered":"2023-03-25T04:27:44.310650+0000","last_clean":"2023-03-25T04:27:44.310650+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:27:44.310650+0000","last_undegraded":"2023-03-25T04:27:44.310650+0000","last_fullsized":"2023-03-25T04:27:44.310650+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":26,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.312962+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:27:44.312962+0000","last_peered":"2023-03-25T04:27:44.312962+0000","last_clean":"2023-03-25T04:27:44.312962+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:27:44.312962+0000","last_undegraded":"2023-03-25T04:27:44.312962+0000","last_fullsized":"2023-03-25T04:27:44.312962+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315064+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:27:45.315064+0000","last_peered":"2023-03-25T04:27:45.315064+0000","last_clean":"2023-03-25T04:27:45.315064+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:27:45.315064+0000","last_undegraded":"2023-03-25T04:27:45.315064+0000","last_fullsized":"2023-03-25T04:27:45.315064+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":25,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.310707+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:27:44.310707+0000","last_peered":"2023-03-25T04:27:44.310707+0000","last_clean":"2023-03-25T04:27:44.310707+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:27:44.310707+0000","last_undegraded":"2023-03-25T04:27:44.310707+0000","last_fullsized":"2023-03-25T04:27:44.310707+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117007,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.375}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051597,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.186}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:28:28.362 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean
, 2023-03-25T04:28:28.362 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:28:45.655 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":124,"stamp":"2023-03-25T04:28:44.111265+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002492"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.314661+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:27:45.314661+0000","last_peered":"2023-03-25T04:27:45.314661+0000","last_clean":"2023-03-25T04:27:45.314661+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:27:45.314661+0000","last_undegraded":"2023-03-25T04:27:45.314661+0000","last_fullsized":"2023-03-25T04:27:45.314661+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":25,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.310745+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:27:44.310745+0000","last_peered":"2023-03-25T04:27:44.310745+0000","last_clean":"2023-03-25T04:27:44.310745+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:27:44.310745+0000","last_undegraded":"2023-03-25T04:27:44.310745+0000","last_fullsized":"2023-03-25T04:27:44.310745+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":25,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.310650+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:27:44.310650+0000","last_peered":"2023-03-25T04:27:44.310650+0000","last_clean":"2023-03-25T04:27:44.310650+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:27:44.310650+0000","last_undegraded":"2023-03-25T04:27:44.310650+0000","last_fullsized":"2023-03-25T04:27:44.310650+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":26,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.312962+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:27:44.312962+0000","last_peered":"2023-03-25T04:27:44.312962+0000","last_clean":"2023-03-25T04:27:44.312962+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:27:44.312962+0000","last_undegraded":"2023-03-25T04:27:44.312962+0000","last_fullsized":"2023-03-25T04:27:44.312962+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315064+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:27:45.315064+0000","last_peered":"2023-03-25T04:27:45.315064+0000","last_clean":"2023-03-25T04:27:45.315064+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:27:45.315064+0000","last_undegraded":"2023-03-25T04:27:45.315064+0000","last_fullsized":"2023-03-25T04:27:45.315064+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":25,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.310707+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:27:44.310707+0000","last_peered":"2023-03-25T04:27:44.310707+0000","last_clean":"2023-03-25T04:27:44.310707+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:27:44.310707+0000","last_undegraded":"2023-03-25T04:27:44.310707+0000","last_fullsized":"2023-03-25T04:27:44.310707+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117011,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:28:31 2023","interfaces":[{"interface":"back","average":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.36899999999999999},{"interface":"front","average":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"min":{"1min":0.19,"5min":0.19,"15min":0.19},"max":{"1min":0.46300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.39800000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051601,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:28:24 2023","interfaces":[{"interface":"back","average":{"1min":0.249,"5min":0.249,"15min":0.249},"min":{"1min":0.17699999999999999,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"last":0.254},{"interface":"front","average":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"min":{"1min":0.20200000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"last":0.312}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:28:45.656 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace: 2023-03-25T04:29:05.171 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}
, 2023-03-25T04:29:05.171 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json
, 2023-03-25T04:29:05.561 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:29:22.465 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:29:22.773 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all

Ignoring malparsed backtrace: 2023-03-25T04:29:42.405 INFO:teuthology.orchestra.run.smithi070.stdout:
, 2023-03-25T04:29:42.406 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":240,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":9,"modified":"2023-03-25T04:29:32.120358+0000","services":{"mon":{"daemons":{"summary":"","a":{"start_epoch":0,"start_stamp":"0.000000","gid":0,"addr":"(unrecognized address family 0)/0","metadata":{},"task_status":{}}}}}},"progress_events":{}}
, 2023-03-25T04:29:42.419 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:29:59.627 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":161,"stamp":"2023-03-25T04:29:58.125186+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002398"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315468+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:27:45.315468+0000","last_peered":"2023-03-25T04:27:45.315468+0000","last_clean":"2023-03-25T04:27:45.315468+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:27:45.315468+0000","last_undegraded":"2023-03-25T04:27:45.315468+0000","last_fullsized":"2023-03-25T04:27:45.315468+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312845+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:27:45.312845+0000","last_peered":"2023-03-25T04:27:45.312845+0000","last_clean":"2023-03-25T04:27:45.312845+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:27:45.312845+0000","last_undegraded":"2023-03-25T04:27:45.312845+0000","last_fullsized":"2023-03-25T04:27:45.312845+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312882+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:27:45.312882+0000","last_peered":"2023-03-25T04:27:45.312882+0000","last_clean":"2023-03-25T04:27:45.312882+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:27:45.312882+0000","last_undegraded":"2023-03-25T04:27:45.312882+0000","last_fullsized":"2023-03-25T04:27:45.312882+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.311730+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:27:45.311730+0000","last_peered":"2023-03-25T04:27:45.311730+0000","last_clean":"2023-03-25T04:27:45.311730+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:27:45.311730+0000","last_undegraded":"2023-03-25T04:27:45.311730+0000","last_fullsized":"2023-03-25T04:27:45.311730+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315735+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:27:45.315735+0000","last_peered":"2023-03-25T04:27:45.315735+0000","last_clean":"2023-03-25T04:27:45.315735+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:27:45.315735+0000","last_undegraded":"2023-03-25T04:27:45.315735+0000","last_fullsized":"2023-03-25T04:27:45.315735+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312833+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:27:45.312833+0000","last_peered":"2023-03-25T04:27:45.312833+0000","last_clean":"2023-03-25T04:27:45.312833+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:27:45.312833+0000","last_undegraded":"2023-03-25T04:27:45.312833+0000","last_fullsized":"2023-03-25T04:27:45.312833+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117026,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:29:33 2023","interfaces":[{"interface":"back","average":{"1min":0.26300000000000001,"5min":0.29099999999999998,"15min":0.29599999999999999},"min":{"1min":0.191,"5min":0.191,"15min":0.191},"max":{"1min":0.371,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.20499999999999999},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.29599999999999999,"15min":0.29999999999999999},"min":{"1min":0.21099999999999999,"5min":0.19,"15min":0.19},"max":{"1min":0.39800000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.19500000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051616,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:29:29 2023","interfaces":[{"interface":"back","average":{"1min":0.27200000000000002,"5min":0.253,"15min":0.25},"min":{"1min":0.183,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"last":0.26400000000000001},{"interface":"front","average":{"1min":0.28499999999999998,"5min":0.26800000000000002,"15min":0.26500000000000001},"min":{"1min":0.16600000000000001,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"last":0.29899999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:29:59.629 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean

Ignoring malparsed backtrace: 2023-03-25T04:30:16.889 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:30:16.903 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":170,"stamp":"2023-03-25T04:30:16.128427+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002222"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315468+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:27:45.315468+0000","last_peered":"2023-03-25T04:27:45.315468+0000","last_clean":"2023-03-25T04:27:45.315468+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:27:45.315468+0000","last_undegraded":"2023-03-25T04:27:45.315468+0000","last_fullsized":"2023-03-25T04:27:45.315468+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312845+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:27:45.312845+0000","last_peered":"2023-03-25T04:27:45.312845+0000","last_clean":"2023-03-25T04:27:45.312845+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:27:45.312845+0000","last_undegraded":"2023-03-25T04:27:45.312845+0000","last_fullsized":"2023-03-25T04:27:45.312845+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312882+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:27:45.312882+0000","last_peered":"2023-03-25T04:27:45.312882+0000","last_clean":"2023-03-25T04:27:45.312882+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:27:45.312882+0000","last_undegraded":"2023-03-25T04:27:45.312882+0000","last_fullsized":"2023-03-25T04:27:45.312882+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.311730+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:27:45.311730+0000","last_peered":"2023-03-25T04:27:45.311730+0000","last_clean":"2023-03-25T04:27:45.311730+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:27:45.311730+0000","last_undegraded":"2023-03-25T04:27:45.311730+0000","last_fullsized":"2023-03-25T04:27:45.311730+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315735+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:27:45.315735+0000","last_peered":"2023-03-25T04:27:45.315735+0000","last_clean":"2023-03-25T04:27:45.315735+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:27:45.315735+0000","last_undegraded":"2023-03-25T04:27:45.315735+0000","last_fullsized":"2023-03-25T04:27:45.315735+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312833+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:27:45.312833+0000","last_peered":"2023-03-25T04:27:45.312833+0000","last_clean":"2023-03-25T04:27:45.312833+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:27:45.312833+0000","last_undegraded":"2023-03-25T04:27:45.312833+0000","last_fullsized":"2023-03-25T04:27:45.312833+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117030,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:29:33 2023","interfaces":[{"interface":"back","average":{"1min":0.26300000000000001,"5min":0.29099999999999998,"15min":0.29599999999999999},"min":{"1min":0.191,"5min":0.191,"15min":0.191},"max":{"1min":0.371,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.245},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.29599999999999999,"15min":0.29999999999999999},"min":{"1min":0.21099999999999999,"5min":0.19,"15min":0.19},"max":{"1min":0.39800000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.20899999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051620,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:29:29 2023","interfaces":[{"interface":"back","average":{"1min":0.27200000000000002,"5min":0.253,"15min":0.25},"min":{"1min":0.183,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"last":0.19400000000000001},{"interface":"front","average":{"1min":0.28499999999999998,"5min":0.26800000000000002,"15min":0.26500000000000001},"min":{"1min":0.16600000000000001,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"last":0.23599999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:30:31.984+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:30:32.201+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:30:32.981+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)

Ignoring malparsed backtrace: 2023-03-25T04:30:48.449 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now
, 2023-03-25T04:30:48.450 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:30:48.760 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:31:08.342 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":196,"stamp":"2023-03-25T04:31:08.139018+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002431"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315468+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:27:45.315468+0000","last_peered":"2023-03-25T04:27:45.315468+0000","last_clean":"2023-03-25T04:27:45.315468+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:27:45.315468+0000","last_undegraded":"2023-03-25T04:27:45.315468+0000","last_fullsized":"2023-03-25T04:27:45.315468+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312845+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:27:45.312845+0000","last_peered":"2023-03-25T04:27:45.312845+0000","last_clean":"2023-03-25T04:27:45.312845+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:27:45.312845+0000","last_undegraded":"2023-03-25T04:27:45.312845+0000","last_fullsized":"2023-03-25T04:27:45.312845+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312882+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:27:45.312882+0000","last_peered":"2023-03-25T04:27:45.312882+0000","last_clean":"2023-03-25T04:27:45.312882+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:27:45.312882+0000","last_undegraded":"2023-03-25T04:27:45.312882+0000","last_fullsized":"2023-03-25T04:27:45.312882+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.311730+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:27:45.311730+0000","last_peered":"2023-03-25T04:27:45.311730+0000","last_clean":"2023-03-25T04:27:45.311730+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:27:45.311730+0000","last_undegraded":"2023-03-25T04:27:45.311730+0000","last_fullsized":"2023-03-25T04:27:45.311730+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315735+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:27:45.315735+0000","last_peered":"2023-03-25T04:27:45.315735+0000","last_clean":"2023-03-25T04:27:45.315735+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:27:45.315735+0000","last_undegraded":"2023-03-25T04:27:45.315735+0000","last_fullsized":"2023-03-25T04:27:45.315735+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312833+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:27:45.312833+0000","last_peered":"2023-03-25T04:27:45.312833+0000","last_clean":"2023-03-25T04:27:45.312833+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:27:45.312833+0000","last_undegraded":"2023-03-25T04:27:45.312833+0000","last_fullsized":"2023-03-25T04:27:45.312833+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117040,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:30:37 2023","interfaces":[{"interface":"back","average":{"1min":0.28799999999999998,"5min":0.28899999999999998,"15min":0.29499999999999998},"min":{"1min":0.20200000000000001,"5min":0.191,"15min":0.191},"max":{"1min":0.433,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.29799999999999999},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.28999999999999998,"15min":0.29799999999999999},"min":{"1min":0.17499999999999999,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":0.40300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.25700000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051630,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:30:32 2023","interfaces":[{"interface":"back","average":{"1min":0.25600000000000001,"5min":0.255,"15min":0.251},"min":{"1min":0.19400000000000001,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.35699999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"last":0.22800000000000001},{"interface":"front","average":{"1min":0.28699999999999998,"5min":0.27200000000000002,"15min":0.26600000000000001},"min":{"1min":0.20599999999999999,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.39400000000000002,"5min":0.42899999999999999,"15min":0.42899999999999999},"last":0.24099999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:31:08.343 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace: 2023-03-25T04:31:24.283+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:31:25.192+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:31:25.295+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)

Ignoring malparsed backtrace: 2023-03-25T04:31:40.257 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:31:40.271 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":212,"stamp":"2023-03-25T04:31:40.144732+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002208"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315468+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:27:45.315468+0000","last_peered":"2023-03-25T04:27:45.315468+0000","last_clean":"2023-03-25T04:27:45.315468+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:27:45.315468+0000","last_undegraded":"2023-03-25T04:27:45.315468+0000","last_fullsized":"2023-03-25T04:27:45.315468+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312845+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:27:45.312845+0000","last_peered":"2023-03-25T04:27:45.312845+0000","last_clean":"2023-03-25T04:27:45.312845+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:27:45.312845+0000","last_undegraded":"2023-03-25T04:27:45.312845+0000","last_fullsized":"2023-03-25T04:27:45.312845+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:23.308607+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:29:23.308607+0000","last_peered":"2023-03-25T04:29:23.308607+0000","last_clean":"2023-03-25T04:29:23.308607+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:29:23.308607+0000","last_undegraded":"2023-03-25T04:29:23.308607+0000","last_fullsized":"2023-03-25T04:29:23.308607+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":28,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:20.554710+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:29:20.554710+0000","last_peered":"2023-03-25T04:29:20.554710+0000","last_clean":"2023-03-25T04:29:20.554710+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:29:20.554710+0000","last_undegraded":"2023-03-25T04:29:20.554710+0000","last_fullsized":"2023-03-25T04:29:20.554710+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309216+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:29:28.309216+0000","last_peered":"2023-03-25T04:29:28.309216+0000","last_clean":"2023-03-25T04:29:28.309216+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:29:28.309216+0000","last_undegraded":"2023-03-25T04:29:28.309216+0000","last_fullsized":"2023-03-25T04:29:28.309216+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309233+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:29:28.309233+0000","last_peered":"2023-03-25T04:29:28.309233+0000","last_clean":"2023-03-25T04:29:28.309233+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:29:28.309233+0000","last_undegraded":"2023-03-25T04:29:28.309233+0000","last_fullsized":"2023-03-25T04:29:28.309233+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117046,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:30:37 2023","interfaces":[{"interface":"back","average":{"1min":0.28799999999999998,"5min":0.28899999999999998,"15min":0.29499999999999998},"min":{"1min":0.20200000000000001,"5min":0.191,"15min":0.191},"max":{"1min":0.433,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.216},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.28999999999999998,"15min":0.29799999999999999},"min":{"1min":0.17499999999999999,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":0.40300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.24299999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051637,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:31:35 2023","interfaces":[{"interface":"back","average":{"1min":0.26100000000000001,"5min":0.25700000000000001,"15min":0.251},"min":{"1min":0.19,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.35899999999999999,"5min":0.41599999999999998,"15min":0.41599999999999998},"last":0.221},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.27400000000000002,"15min":0.26700000000000002},"min":{"1min":0.20200000000000001,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.374,"5min":0.42899999999999999,"15min":0.42899999999999999},"last":0.24099999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:32:01.906+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:32:02.388+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:32:02.794 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:32:22.326 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":233,"stamp":"2023-03-25T04:32:22.153289+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002545"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:33.309748+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:29:33.309748+0000","last_peered":"2023-03-25T04:29:33.309748+0000","last_clean":"2023-03-25T04:29:33.309748+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:29:33.309748+0000","last_undegraded":"2023-03-25T04:29:33.309748+0000","last_fullsized":"2023-03-25T04:29:33.309748+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:33.309729+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:29:33.309729+0000","last_peered":"2023-03-25T04:29:33.309729+0000","last_clean":"2023-03-25T04:29:33.309729+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:29:33.309729+0000","last_undegraded":"2023-03-25T04:29:33.309729+0000","last_fullsized":"2023-03-25T04:29:33.309729+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:23.308607+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:29:23.308607+0000","last_peered":"2023-03-25T04:29:23.308607+0000","last_clean":"2023-03-25T04:29:23.308607+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:29:23.308607+0000","last_undegraded":"2023-03-25T04:29:23.308607+0000","last_fullsized":"2023-03-25T04:29:23.308607+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":28,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:20.554710+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:29:20.554710+0000","last_peered":"2023-03-25T04:29:20.554710+0000","last_clean":"2023-03-25T04:29:20.554710+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:29:20.554710+0000","last_undegraded":"2023-03-25T04:29:20.554710+0000","last_fullsized":"2023-03-25T04:29:20.554710+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309216+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:29:28.309216+0000","last_peered":"2023-03-25T04:29:28.309216+0000","last_clean":"2023-03-25T04:29:28.309216+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:29:28.309216+0000","last_undegraded":"2023-03-25T04:29:28.309216+0000","last_fullsized":"2023-03-25T04:29:28.309216+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309233+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:29:28.309233+0000","last_peered":"2023-03-25T04:29:28.309233+0000","last_clean":"2023-03-25T04:29:28.309233+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:29:28.309233+0000","last_undegraded":"2023-03-25T04:29:28.309233+0000","last_fullsized":"2023-03-25T04:29:28.309233+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117055,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:31:42 2023","interfaces":[{"interface":"back","average":{"1min":0.27500000000000002,"5min":0.28399999999999997,"15min":0.29399999999999998},"min":{"1min":0.19800000000000001,"5min":0.191,"15min":0.191},"max":{"1min":0.36899999999999999,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.32300000000000001},{"interface":"front","average":{"1min":0.27900000000000003,"5min":0.28499999999999998,"15min":0.29599999999999999},"min":{"1min":0.216,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":0.40799999999999997,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.27100000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051646,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:31:35 2023","interfaces":[{"interface":"back","average":{"1min":0.26100000000000001,"5min":0.25700000000000001,"15min":0.251},"min":{"1min":0.19,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.35899999999999999,"5min":0.41599999999999998,"15min":0.41599999999999998},"last":0.25600000000000001},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.27400000000000002,"15min":0.26700000000000002},"min":{"1min":0.20200000000000001,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.374,"5min":0.42899999999999999,"15min":0.42899999999999999},"last":0.29199999999999998}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:32:22.327 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean
, 2023-03-25T04:32:22.327 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}

Ignoring malparsed backtrace: 2023-03-25T04:32:39.836 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:32:39.849 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":241,"stamp":"2023-03-25T04:32:38.156312+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002346"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:33.309748+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:29:33.309748+0000","last_peered":"2023-03-25T04:29:33.309748+0000","last_clean":"2023-03-25T04:29:33.309748+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:29:33.309748+0000","last_undegraded":"2023-03-25T04:29:33.309748+0000","last_fullsized":"2023-03-25T04:29:33.309748+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:33.309729+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:29:33.309729+0000","last_peered":"2023-03-25T04:29:33.309729+0000","last_clean":"2023-03-25T04:29:33.309729+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:29:33.309729+0000","last_undegraded":"2023-03-25T04:29:33.309729+0000","last_fullsized":"2023-03-25T04:29:33.309729+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:23.308607+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:29:23.308607+0000","last_peered":"2023-03-25T04:29:23.308607+0000","last_clean":"2023-03-25T04:29:23.308607+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:29:23.308607+0000","last_undegraded":"2023-03-25T04:29:23.308607+0000","last_fullsized":"2023-03-25T04:29:23.308607+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":28,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:20.554710+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:29:20.554710+0000","last_peered":"2023-03-25T04:29:20.554710+0000","last_clean":"2023-03-25T04:29:20.554710+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:29:20.554710+0000","last_undegraded":"2023-03-25T04:29:20.554710+0000","last_fullsized":"2023-03-25T04:29:20.554710+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309216+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:29:28.309216+0000","last_peered":"2023-03-25T04:29:28.309216+0000","last_clean":"2023-03-25T04:29:28.309216+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:29:28.309216+0000","last_undegraded":"2023-03-25T04:29:28.309216+0000","last_fullsized":"2023-03-25T04:29:28.309216+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309233+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:29:28.309233+0000","last_peered":"2023-03-25T04:29:28.309233+0000","last_clean":"2023-03-25T04:29:28.309233+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:29:28.309233+0000","last_undegraded":"2023-03-25T04:29:28.309233+0000","last_fullsized":"2023-03-25T04:29:28.309233+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117058,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:31:42 2023","interfaces":[{"interface":"back","average":{"1min":0.27500000000000002,"5min":0.28399999999999997,"15min":0.29399999999999998},"min":{"1min":0.19800000000000001,"5min":0.191,"15min":0.191},"max":{"1min":0.36899999999999999,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.30599999999999999},{"interface":"front","average":{"1min":0.27900000000000003,"5min":0.28499999999999998,"15min":0.29599999999999999},"min":{"1min":0.216,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":0.40799999999999997,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.32500000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051649,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:31:35 2023","interfaces":[{"interface":"back","average":{"1min":0.26100000000000001,"5min":0.25700000000000001,"15min":0.251},"min":{"1min":0.19,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.35899999999999999,"5min":0.41599999999999998,"15min":0.41599999999999998},"last":0.24199999999999999},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.27400000000000002,"15min":0.26700000000000002},"min":{"1min":0.20200000000000001,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.374,"5min":0.42899999999999999,"15min":0.42899999999999999},"last":0.22600000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:32:54.944+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:32:55.469+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:32:55.927+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)

Ignoring malparsed backtrace: 2023-03-25T04:33:13.958 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean
, 2023-03-25T04:33:13.959 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}
, 2023-03-25T04:33:13.959 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json

Ignoring malparsed backtrace: 2023-03-25T04:33:31.199 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":267,"stamp":"2023-03-25T04:33:30.166660+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002269"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:33.309748+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:29:33.309748+0000","last_peered":"2023-03-25T04:29:33.309748+0000","last_clean":"2023-03-25T04:29:33.309748+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:29:33.309748+0000","last_undegraded":"2023-03-25T04:29:33.309748+0000","last_fullsized":"2023-03-25T04:29:33.309748+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:33.309729+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:29:33.309729+0000","last_peered":"2023-03-25T04:29:33.309729+0000","last_clean":"2023-03-25T04:29:33.309729+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:29:33.309729+0000","last_undegraded":"2023-03-25T04:29:33.309729+0000","last_fullsized":"2023-03-25T04:29:33.309729+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":28,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:31:23.320894+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:31:23.320894+0000","last_peered":"2023-03-25T04:31:23.320894+0000","last_clean":"2023-03-25T04:31:23.320894+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:31:23.320894+0000","last_undegraded":"2023-03-25T04:31:23.320894+0000","last_fullsized":"2023-03-25T04:31:23.320894+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":28,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:20.554710+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:29:20.554710+0000","last_peered":"2023-03-25T04:29:20.554710+0000","last_clean":"2023-03-25T04:29:20.554710+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:29:20.554710+0000","last_undegraded":"2023-03-25T04:29:20.554710+0000","last_fullsized":"2023-03-25T04:29:20.554710+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309216+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:29:28.309216+0000","last_peered":"2023-03-25T04:29:28.309216+0000","last_clean":"2023-03-25T04:29:28.309216+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:29:28.309216+0000","last_undegraded":"2023-03-25T04:29:28.309216+0000","last_fullsized":"2023-03-25T04:29:28.309216+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309233+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:29:28.309233+0000","last_peered":"2023-03-25T04:29:28.309233+0000","last_clean":"2023-03-25T04:29:28.309233+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:29:28.309233+0000","last_undegraded":"2023-03-25T04:29:28.309233+0000","last_fullsized":"2023-03-25T04:29:28.309233+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117068,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:32:46 2023","interfaces":[{"interface":"back","average":{"1min":0.27000000000000002,"5min":0.27900000000000003,"15min":0.29199999999999998},"min":{"1min":0.17899999999999999,"5min":0.17899999999999999,"15min":0.17899999999999999},"max":{"1min":0.35599999999999998,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.28799999999999998},{"interface":"front","average":{"1min":0.26800000000000002,"5min":0.27800000000000002,"15min":0.29399999999999998},"min":{"1min":0.19900000000000001,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":0.377,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.315}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051660,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:32:36 2023","interfaces":[{"interface":"back","average":{"1min":0.248,"5min":0.25700000000000001,"15min":0.251},"min":{"1min":0.17000000000000001,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"last":0.22700000000000001},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.27600000000000002,"15min":0.26800000000000002},"min":{"1min":0.192,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"last":0.20399999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:33:31.201 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:33:31.400+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)

Ignoring malparsed backtrace: 2023-03-25T04:33:47.531+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:33:47.951+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:33:48.505+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)

Ignoring malparsed backtrace: 2023-03-25T04:34:06.104 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json
, 2023-03-25T04:34:06.118+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:34:06.497 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:34:23.399 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:34:23.665+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:34:23.712 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:34:40.195+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:34:40.523+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:34:40.676 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace: 2023-03-25T04:35:03.381+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:35:04.139+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:35:04.391+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)

Ignoring malparsed backtrace: 2023-03-25T04:35:22.738 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean
, 2023-03-25T04:35:22.739 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}
, 2023-03-25T04:35:22.739 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json

Ignoring malparsed backtrace: 2023-03-25T04:35:41.393+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:35:42.352+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:35:42.376+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:35:57.384 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":340,"stamp":"2023-03-25T04:35:56.194992+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.001930"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":36,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:31:43.322894+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:31:43.322894+0000","last_peered":"2023-03-25T04:31:43.322894+0000","last_clean":"2023-03-25T04:31:43.322894+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:31:43.322894+0000","last_undegraded":"2023-03-25T04:31:43.322894+0000","last_fullsized":"2023-03-25T04:31:43.322894+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":28,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:31:43.322876+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:31:43.322876+0000","last_peered":"2023-03-25T04:31:43.322876+0000","last_clean":"2023-03-25T04:31:43.322876+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:31:43.322876+0000","last_undegraded":"2023-03-25T04:31:43.322876+0000","last_fullsized":"2023-03-25T04:31:43.322876+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:23.331998+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:33:23.331998+0000","last_peered":"2023-03-25T04:33:23.331998+0000","last_clean":"2023-03-25T04:33:23.331998+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:33:23.331998+0000","last_undegraded":"2023-03-25T04:33:23.331998+0000","last_fullsized":"2023-03-25T04:33:23.331998+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:30.576723+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:33:30.576723+0000","last_peered":"2023-03-25T04:33:30.576723+0000","last_clean":"2023-03-25T04:33:30.576723+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:33:30.576723+0000","last_undegraded":"2023-03-25T04:33:30.576723+0000","last_fullsized":"2023-03-25T04:33:30.576723+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":37,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:38.333447+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:33:38.333447+0000","last_peered":"2023-03-25T04:33:38.333447+0000","last_clean":"2023-03-25T04:33:38.333447+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:33:38.333447+0000","last_undegraded":"2023-03-25T04:33:38.333447+0000","last_fullsized":"2023-03-25T04:33:38.333447+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:38.333467+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:33:38.333467+0000","last_peered":"2023-03-25T04:33:38.333467+0000","last_clean":"2023-03-25T04:33:38.333467+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:33:38.333467+0000","last_undegraded":"2023-03-25T04:33:38.333467+0000","last_fullsized":"2023-03-25T04:33:38.333467+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117098,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:35:49 2023","interfaces":[{"interface":"back","average":{"1min":0.255,"5min":0.26000000000000001,"15min":0.28199999999999997},"min":{"1min":0.17899999999999999,"5min":0.16,"15min":0.16},"max":{"1min":0.36399999999999999,"5min":0.42599999999999999,"15min":0.45100000000000001},"last":0.26100000000000001},{"interface":"front","average":{"1min":0.26000000000000001,"5min":0.26700000000000002,"15min":0.28599999999999998},"min":{"1min":0.188,"5min":0.17999999999999999,"15min":0.17499999999999999},"max":{"1min":0.39100000000000001,"5min":0.45200000000000001,"15min":0.46300000000000002},"last":0.28399999999999997}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051689,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:35:42 2023","interfaces":[{"interface":"back","average":{"1min":0.29399999999999998,"5min":0.27200000000000002,"15min":0.25800000000000001},"min":{"1min":0.17299999999999999,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.41099999999999998,"5min":0.42699999999999999,"15min":0.42699999999999999},"last":0.20399999999999999},{"interface":"front","average":{"1min":0.30299999999999999,"5min":0.28399999999999997,"15min":0.27300000000000002},"min":{"1min":0.19700000000000001,"5min":0.192,"15min":0.16600000000000001},"max":{"1min":0.43099999999999999,"5min":0.45800000000000002,"15min":0.45800000000000002},"last":0.22}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:35:57.624+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)

Ignoring malparsed backtrace: 2023-03-25T04:36:17.226 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:36:17.239 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":350,"stamp":"2023-03-25T04:36:16.198866+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002148"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":37,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:53.335026+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:33:53.335026+0000","last_peered":"2023-03-25T04:33:53.335026+0000","last_clean":"2023-03-25T04:33:53.335026+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:33:53.335026+0000","last_undegraded":"2023-03-25T04:33:53.335026+0000","last_fullsized":"2023-03-25T04:33:53.335026+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:53.335009+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:33:53.335009+0000","last_peered":"2023-03-25T04:33:53.335009+0000","last_clean":"2023-03-25T04:33:53.335009+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:33:53.335009+0000","last_undegraded":"2023-03-25T04:33:53.335009+0000","last_fullsized":"2023-03-25T04:33:53.335009+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:23.331998+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:33:23.331998+0000","last_peered":"2023-03-25T04:33:23.331998+0000","last_clean":"2023-03-25T04:33:23.331998+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:33:23.331998+0000","last_undegraded":"2023-03-25T04:33:23.331998+0000","last_fullsized":"2023-03-25T04:33:23.331998+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:30.576723+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:33:30.576723+0000","last_peered":"2023-03-25T04:33:30.576723+0000","last_clean":"2023-03-25T04:33:30.576723+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:33:30.576723+0000","last_undegraded":"2023-03-25T04:33:30.576723+0000","last_fullsized":"2023-03-25T04:33:30.576723+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":37,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:38.333447+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:33:38.333447+0000","last_peered":"2023-03-25T04:33:38.333447+0000","last_clean":"2023-03-25T04:33:38.333447+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:33:38.333447+0000","last_undegraded":"2023-03-25T04:33:38.333447+0000","last_fullsized":"2023-03-25T04:33:38.333447+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:38.333467+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:33:38.333467+0000","last_peered":"2023-03-25T04:33:38.333467+0000","last_clean":"2023-03-25T04:33:38.333467+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:33:38.333467+0000","last_undegraded":"2023-03-25T04:33:38.333467+0000","last_fullsized":"2023-03-25T04:33:38.333467+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117102,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:35:49 2023","interfaces":[{"interface":"back","average":{"1min":0.255,"5min":0.26000000000000001,"15min":0.28199999999999997},"min":{"1min":0.17899999999999999,"5min":0.16,"15min":0.16},"max":{"1min":0.36399999999999999,"5min":0.42599999999999999,"15min":0.45100000000000001},"last":0.28399999999999997},{"interface":"front","average":{"1min":0.26000000000000001,"5min":0.26700000000000002,"15min":0.28599999999999998},"min":{"1min":0.188,"5min":0.17999999999999999,"15min":0.17499999999999999},"max":{"1min":0.39100000000000001,"5min":0.45200000000000001,"15min":0.46300000000000002},"last":0.30299999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051693,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:35:42 2023","interfaces":[{"interface":"back","average":{"1min":0.29399999999999998,"5min":0.27200000000000002,"15min":0.25800000000000001},"min":{"1min":0.17299999999999999,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.41099999999999998,"5min":0.42699999999999999,"15min":0.42699999999999999},"last":0.35099999999999998},{"interface":"front","average":{"1min":0.30299999999999999,"5min":0.28399999999999997,"15min":0.27300000000000002},"min":{"1min":0.19700000000000001,"5min":0.192,"15min":0.16600000000000001},"max":{"1min":0.43099999999999999,"5min":0.45800000000000002,"15min":0.45800000000000002},"last":0.32900000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:36:34.556+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:36:34.717+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:36:35.583+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)

Ignoring malparsed backtrace: 2023-03-25T04:36:52.680 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json
, 2023-03-25T04:36:52.701+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:36:53.067 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:37:09.963 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:37:10.273 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all

Ignoring malparsed backtrace: 2023-03-25T04:37:26.856+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:37:27.288 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:37:27.538+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:37:41.903 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":392,"stamp":"2023-03-25T04:37:40.213373+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002012"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":37,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:53.335026+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:33:53.335026+0000","last_peered":"2023-03-25T04:33:53.335026+0000","last_clean":"2023-03-25T04:33:53.335026+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:33:53.335026+0000","last_undegraded":"2023-03-25T04:33:53.335026+0000","last_fullsized":"2023-03-25T04:33:53.335026+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:53.335009+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:33:53.335009+0000","last_peered":"2023-03-25T04:33:53.335009+0000","last_clean":"2023-03-25T04:33:53.335009+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:33:53.335009+0000","last_undegraded":"2023-03-25T04:33:53.335009+0000","last_fullsized":"2023-03-25T04:33:53.335009+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:23.343585+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:35:23.343585+0000","last_peered":"2023-03-25T04:35:23.343585+0000","last_clean":"2023-03-25T04:35:23.343585+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:35:23.343585+0000","last_undegraded":"2023-03-25T04:35:23.343585+0000","last_fullsized":"2023-03-25T04:35:23.343585+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:30.576723+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:33:30.576723+0000","last_peered":"2023-03-25T04:33:30.576723+0000","last_clean":"2023-03-25T04:33:30.576723+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:33:30.576723+0000","last_undegraded":"2023-03-25T04:33:30.576723+0000","last_fullsized":"2023-03-25T04:33:30.576723+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":37,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:38.333447+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:33:38.333447+0000","last_peered":"2023-03-25T04:33:38.333447+0000","last_clean":"2023-03-25T04:33:38.333447+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:33:38.333447+0000","last_undegraded":"2023-03-25T04:33:38.333447+0000","last_fullsized":"2023-03-25T04:33:38.333447+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:38.333467+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:33:38.333467+0000","last_peered":"2023-03-25T04:33:38.333467+0000","last_clean":"2023-03-25T04:33:38.333467+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:33:38.333467+0000","last_undegraded":"2023-03-25T04:33:38.333467+0000","last_fullsized":"2023-03-25T04:33:38.333467+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117119,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:36:50 2023","interfaces":[{"interface":"back","average":{"1min":0.254,"5min":0.25600000000000001,"15min":0.27900000000000003},"min":{"1min":0.20499999999999999,"5min":0.16,"15min":0.16},"max":{"1min":0.371,"5min":0.42599999999999999,"15min":0.45100000000000001},"last":0.23000000000000001},{"interface":"front","average":{"1min":0.26800000000000002,"5min":0.26500000000000001,"15min":0.28399999999999997},"min":{"1min":0.19700000000000001,"5min":0.17999999999999999,"15min":0.17499999999999999},"max":{"1min":0.34699999999999998,"5min":0.45200000000000001,"15min":0.46300000000000002},"last":0.20899999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051710,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:36:44 2023","interfaces":[{"interface":"back","average":{"1min":0.28399999999999997,"5min":0.27700000000000002,"15min":0.26100000000000001},"min":{"1min":0.192,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.40899999999999997,"5min":0.42699999999999999,"15min":0.42699999999999999},"last":0.39700000000000002},{"interface":"front","average":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.27500000000000002},"min":{"1min":0.182,"5min":0.182,"15min":0.16600000000000001},"max":{"1min":0.439,"5min":0.45800000000000002,"15min":0.45800000000000002},"last":0.439}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:37:42.400+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)

Ignoring malparsed backtrace: 2023-03-25T04:38:04.766+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:38:04.922 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:38:05.246 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:38:19.605 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":411,"stamp":"2023-03-25T04:38:18.220966+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002368"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":38,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:36:03.347070+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:36:03.347070+0000","last_peered":"2023-03-25T04:36:03.347070+0000","last_clean":"2023-03-25T04:36:03.347070+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:36:03.347070+0000","last_undegraded":"2023-03-25T04:36:03.347070+0000","last_fullsized":"2023-03-25T04:36:03.347070+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:36:03.347025+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:36:03.347025+0000","last_peered":"2023-03-25T04:36:03.347025+0000","last_clean":"2023-03-25T04:36:03.347025+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:36:03.347025+0000","last_undegraded":"2023-03-25T04:36:03.347025+0000","last_fullsized":"2023-03-25T04:36:03.347025+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:23.343585+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:35:23.343585+0000","last_peered":"2023-03-25T04:35:23.343585+0000","last_clean":"2023-03-25T04:35:23.343585+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:35:23.343585+0000","last_undegraded":"2023-03-25T04:35:23.343585+0000","last_fullsized":"2023-03-25T04:35:23.343585+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:35.587025+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:35:35.587025+0000","last_peered":"2023-03-25T04:35:35.587025+0000","last_clean":"2023-03-25T04:35:35.587025+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:35:35.587025+0000","last_undegraded":"2023-03-25T04:35:35.587025+0000","last_fullsized":"2023-03-25T04:35:35.587025+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":38,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:43.345308+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:35:43.345308+0000","last_peered":"2023-03-25T04:35:43.345308+0000","last_clean":"2023-03-25T04:35:43.345308+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:35:43.345308+0000","last_undegraded":"2023-03-25T04:35:43.345308+0000","last_fullsized":"2023-03-25T04:35:43.345308+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:43.345324+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:35:43.345324+0000","last_peered":"2023-03-25T04:35:43.345324+0000","last_clean":"2023-03-25T04:35:43.345324+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:35:43.345324+0000","last_undegraded":"2023-03-25T04:35:43.345324+0000","last_fullsized":"2023-03-25T04:35:43.345324+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117127,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:37:53 2023","interfaces":[{"interface":"back","average":{"1min":0.25800000000000001,"5min":0.253,"15min":0.27700000000000002},"min":{"1min":0.184,"5min":0.16,"15min":0.16},"max":{"1min":0.44600000000000001,"5min":0.44600000000000001,"15min":0.45100000000000001},"last":0.35199999999999998},{"interface":"front","average":{"1min":0.26900000000000002,"5min":0.26500000000000001,"15min":0.28199999999999997},"min":{"1min":0.192,"5min":0.17999999999999999,"15min":0.17499999999999999},"max":{"1min":0.46400000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"last":0.33200000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051717,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:37:44 2023","interfaces":[{"interface":"back","average":{"1min":0.27300000000000002,"5min":0.28199999999999997,"15min":0.26200000000000001},"min":{"1min":0.17899999999999999,"5min":0.17299999999999999,"15min":0.17000000000000001},"max":{"1min":0.41799999999999998,"5min":0.42099999999999999,"15min":0.42699999999999999},"last":0.28699999999999998},{"interface":"front","average":{"1min":0.29399999999999998,"5min":0.29099999999999998,"15min":0.27700000000000002},"min":{"1min":0.19900000000000001,"5min":0.182,"15min":0.16600000000000001},"max":{"1min":0.441,"5min":0.441,"15min":0.45800000000000002},"last":0.26300000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:38:19.718+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:38:20.315+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)

Ignoring malparsed backtrace: 2023-03-25T04:38:44.283 INFO:teuthology.orchestra.run.smithi070.stdout:
, 2023-03-25T04:38:44.283 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":781,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":16,"modified":"2023-03-25T04:38:40.225321+0000","services":{}},"progress_events":{}}
, 2023-03-25T04:38:44.296 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now

Ignoring malparsed backtrace: 2023-03-25T04:39:02.477+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:39:02.578+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:39:03.524+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:39:20.359 INFO:teuthology.orchestra.run.smithi070.stdout:
, 2023-03-25T04:39:20.359 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":818,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":16,"modified":"2023-03-25T04:38:40.225321+0000","services":{}},"progress_events":{}}
, 2023-03-25T04:39:20.372 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now

Ignoring malparsed backtrace: 2023-03-25T04:39:37.527 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:39:37.541 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":450,"stamp":"2023-03-25T04:39:36.235332+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002086"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":38,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:36:03.347070+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:36:03.347070+0000","last_peered":"2023-03-25T04:36:03.347070+0000","last_clean":"2023-03-25T04:36:03.347070+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:36:03.347070+0000","last_undegraded":"2023-03-25T04:36:03.347070+0000","last_fullsized":"2023-03-25T04:36:03.347070+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:36:03.347025+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:36:03.347025+0000","last_peered":"2023-03-25T04:36:03.347025+0000","last_clean":"2023-03-25T04:36:03.347025+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:36:03.347025+0000","last_undegraded":"2023-03-25T04:36:03.347025+0000","last_fullsized":"2023-03-25T04:36:03.347025+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:23.353778+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:37:23.353778+0000","last_peered":"2023-03-25T04:37:23.353778+0000","last_clean":"2023-03-25T04:37:23.353778+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:37:23.353778+0000","last_undegraded":"2023-03-25T04:37:23.353778+0000","last_fullsized":"2023-03-25T04:37:23.353778+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:35.587025+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:35:35.587025+0000","last_peered":"2023-03-25T04:35:35.587025+0000","last_clean":"2023-03-25T04:35:35.587025+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:35:35.587025+0000","last_undegraded":"2023-03-25T04:35:35.587025+0000","last_fullsized":"2023-03-25T04:35:35.587025+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":38,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:43.345308+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:35:43.345308+0000","last_peered":"2023-03-25T04:35:43.345308+0000","last_clean":"2023-03-25T04:35:43.345308+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:35:43.345308+0000","last_undegraded":"2023-03-25T04:35:43.345308+0000","last_fullsized":"2023-03-25T04:35:43.345308+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:43.345324+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:35:43.345324+0000","last_peered":"2023-03-25T04:35:43.345324+0000","last_clean":"2023-03-25T04:35:43.345324+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:35:43.345324+0000","last_undegraded":"2023-03-25T04:35:43.345324+0000","last_fullsized":"2023-03-25T04:35:43.345324+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117143,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:38:54 2023","interfaces":[{"interface":"back","average":{"1min":0.25800000000000001,"5min":0.25700000000000001,"15min":0.27400000000000002},"min":{"1min":0.192,"5min":0.17399999999999999,"15min":0.16},"max":{"1min":0.39400000000000002,"5min":0.44600000000000001,"15min":0.45100000000000001},"last":0.26500000000000001},{"interface":"front","average":{"1min":0.25900000000000001,"5min":0.26700000000000002,"15min":0.27900000000000003},"min":{"1min":0.19500000000000001,"5min":0.188,"15min":0.17499999999999999},"max":{"1min":0.41499999999999998,"5min":0.46400000000000002,"15min":0.46400000000000002},"last":0.29099999999999998}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051733,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:38:45 2023","interfaces":[{"interface":"back","average":{"1min":0.27300000000000002,"5min":0.28100000000000003,"15min":0.26400000000000001},"min":{"1min":0.19400000000000001,"5min":0.17299999999999999,"15min":0.17000000000000001},"max":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"last":0.27300000000000002},{"interface":"front","average":{"1min":0.28799999999999998,"5min":0.29099999999999998,"15min":0.27800000000000002},"min":{"1min":0.19800000000000001,"5min":0.182,"15min":0.16600000000000001},"max":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"last":0.308}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:39:54.416+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:39:54.455 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:39:54.780 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:40:09.288+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:40:09.389+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:40:10.267+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)

Ignoring malparsed backtrace: 2023-03-25T04:40:28.984 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":476,"stamp":"2023-03-25T04:40:28.245911+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002693"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":39,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:38:13.358968+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:38:13.358968+0000","last_peered":"2023-03-25T04:38:13.358968+0000","last_clean":"2023-03-25T04:38:13.358968+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:38:13.358968+0000","last_undegraded":"2023-03-25T04:38:13.358968+0000","last_fullsized":"2023-03-25T04:38:13.358968+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:38:13.358922+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:38:13.358922+0000","last_peered":"2023-03-25T04:38:13.358922+0000","last_clean":"2023-03-25T04:38:13.358922+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:38:13.358922+0000","last_undegraded":"2023-03-25T04:38:13.358922+0000","last_fullsized":"2023-03-25T04:38:13.358922+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:23.353778+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:37:23.353778+0000","last_peered":"2023-03-25T04:37:23.353778+0000","last_clean":"2023-03-25T04:37:23.353778+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:37:23.353778+0000","last_undegraded":"2023-03-25T04:37:23.353778+0000","last_fullsized":"2023-03-25T04:37:23.353778+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:40.598579+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:37:40.598579+0000","last_peered":"2023-03-25T04:37:40.598579+0000","last_clean":"2023-03-25T04:37:40.598579+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:37:40.598579+0000","last_undegraded":"2023-03-25T04:37:40.598579+0000","last_fullsized":"2023-03-25T04:37:40.598579+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":39,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:48.356344+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:37:48.356344+0000","last_peered":"2023-03-25T04:37:48.356344+0000","last_clean":"2023-03-25T04:37:48.356344+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:37:48.356344+0000","last_undegraded":"2023-03-25T04:37:48.356344+0000","last_fullsized":"2023-03-25T04:37:48.356344+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:48.356365+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:37:48.356365+0000","last_peered":"2023-03-25T04:37:48.356365+0000","last_clean":"2023-03-25T04:37:48.356365+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:37:48.356365+0000","last_undegraded":"2023-03-25T04:37:48.356365+0000","last_fullsized":"2023-03-25T04:37:48.356365+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117153,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:39:54 2023","interfaces":[{"interface":"back","average":{"1min":0.25900000000000001,"5min":0.25600000000000001,"15min":0.27100000000000002},"min":{"1min":0.188,"5min":0.17899999999999999,"15min":0.16},"max":{"1min":0.41999999999999998,"5min":0.44600000000000001,"15min":0.45100000000000001},"last":0.255},{"interface":"front","average":{"1min":0.27500000000000002,"5min":0.26600000000000001,"15min":0.27700000000000002},"min":{"1min":0.17699999999999999,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.45600000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"last":0.26800000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051743,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:39:47 2023","interfaces":[{"interface":"back","average":{"1min":0.26300000000000001,"5min":0.27700000000000002,"15min":0.26500000000000001},"min":{"1min":0.187,"5min":0.17299999999999999,"15min":0.17000000000000001},"max":{"1min":0.433,"5min":0.433,"15min":0.433},"last":0.36599999999999999},{"interface":"front","average":{"1min":0.27100000000000002,"5min":0.28799999999999998,"15min":0.27900000000000003},"min":{"1min":0.20100000000000001,"5min":0.182,"15min":0.16600000000000001},"max":{"1min":0.48599999999999999,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.34999999999999998}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:40:28.986 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean
, 2023-03-25T04:40:28.986 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}

Ignoring malparsed backtrace: 2023-03-25T04:40:47.229+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:40:47.503+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:40:48.251+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)

Ignoring malparsed backtrace: 2023-03-25T04:41:03.267 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:41:03.578 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all

Ignoring malparsed backtrace: 2023-03-25T04:41:21.347 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":502,"stamp":"2023-03-25T04:41:20.256204+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002342"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":39,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:38:13.358968+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:38:13.358968+0000","last_peered":"2023-03-25T04:38:13.358968+0000","last_clean":"2023-03-25T04:38:13.358968+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:38:13.358968+0000","last_undegraded":"2023-03-25T04:38:13.358968+0000","last_fullsized":"2023-03-25T04:38:13.358968+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:38:13.358922+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:38:13.358922+0000","last_peered":"2023-03-25T04:38:13.358922+0000","last_clean":"2023-03-25T04:38:13.358922+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:38:13.358922+0000","last_undegraded":"2023-03-25T04:38:13.358922+0000","last_fullsized":"2023-03-25T04:38:13.358922+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:23.353778+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:37:23.353778+0000","last_peered":"2023-03-25T04:37:23.353778+0000","last_clean":"2023-03-25T04:37:23.353778+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:37:23.353778+0000","last_undegraded":"2023-03-25T04:37:23.353778+0000","last_fullsized":"2023-03-25T04:37:23.353778+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:40.598579+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:37:40.598579+0000","last_peered":"2023-03-25T04:37:40.598579+0000","last_clean":"2023-03-25T04:37:40.598579+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:37:40.598579+0000","last_undegraded":"2023-03-25T04:37:40.598579+0000","last_fullsized":"2023-03-25T04:37:40.598579+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":39,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:48.356344+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:37:48.356344+0000","last_peered":"2023-03-25T04:37:48.356344+0000","last_clean":"2023-03-25T04:37:48.356344+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:37:48.356344+0000","last_undegraded":"2023-03-25T04:37:48.356344+0000","last_fullsized":"2023-03-25T04:37:48.356344+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:48.356365+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:37:48.356365+0000","last_peered":"2023-03-25T04:37:48.356365+0000","last_clean":"2023-03-25T04:37:48.356365+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:37:48.356365+0000","last_undegraded":"2023-03-25T04:37:48.356365+0000","last_fullsized":"2023-03-25T04:37:48.356365+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117163,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:40:56 2023","interfaces":[{"interface":"back","average":{"1min":0.28799999999999998,"5min":0.26300000000000001,"15min":0.27100000000000002},"min":{"1min":0.188,"5min":0.184,"15min":0.16},"max":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.17499999999999999},{"interface":"front","average":{"1min":0.29599999999999999,"5min":0.27300000000000002,"15min":0.27700000000000002},"min":{"1min":0.20599999999999999,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.43099999999999999,"5min":0.46400000000000002,"15min":0.46400000000000002},"last":0.20699999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051754,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:40:52 2023","interfaces":[{"interface":"back","average":{"1min":0.28599999999999998,"5min":0.27500000000000002,"15min":0.26700000000000002},"min":{"1min":0.17899999999999999,"5min":0.17899999999999999,"15min":0.17000000000000001},"max":{"1min":0.44,"5min":0.44,"15min":0.44},"last":0.23699999999999999},{"interface":"front","average":{"1min":0.29799999999999999,"5min":0.28699999999999998,"15min":0.28100000000000003},"min":{"1min":0.20300000000000001,"5min":0.182,"15min":0.16600000000000001},"max":{"1min":0.47599999999999998,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.28000000000000003}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:41:21.349 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean
, 2023-03-25T04:41:21.349 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}

Ignoring malparsed backtrace: 2023-03-25T04:41:39.867+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:41:40.383+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:41:40.905+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:41:58.219 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json
, 2023-03-25T04:41:58.319+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:41:58.644 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:42:18.193 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":530,"stamp":"2023-03-25T04:42:16.266659+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002356"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":39,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:38:13.358968+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:38:13.358968+0000","last_peered":"2023-03-25T04:38:13.358968+0000","last_clean":"2023-03-25T04:38:13.358968+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:38:13.358968+0000","last_undegraded":"2023-03-25T04:38:13.358968+0000","last_fullsized":"2023-03-25T04:38:13.358968+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:38:13.358922+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:38:13.358922+0000","last_peered":"2023-03-25T04:38:13.358922+0000","last_clean":"2023-03-25T04:38:13.358922+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:38:13.358922+0000","last_undegraded":"2023-03-25T04:38:13.358922+0000","last_fullsized":"2023-03-25T04:38:13.358922+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:23.364593+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:39:23.364593+0000","last_peered":"2023-03-25T04:39:23.364593+0000","last_clean":"2023-03-25T04:39:23.364593+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:39:23.364593+0000","last_undegraded":"2023-03-25T04:39:23.364593+0000","last_fullsized":"2023-03-25T04:39:23.364593+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:45.608475+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:39:45.608475+0000","last_peered":"2023-03-25T04:39:45.608475+0000","last_clean":"2023-03-25T04:39:45.608475+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:39:45.608475+0000","last_undegraded":"2023-03-25T04:39:45.608475+0000","last_fullsized":"2023-03-25T04:39:45.608475+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":40,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:53.367398+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:39:53.367398+0000","last_peered":"2023-03-25T04:39:53.367398+0000","last_clean":"2023-03-25T04:39:53.367398+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:39:53.367398+0000","last_undegraded":"2023-03-25T04:39:53.367398+0000","last_fullsized":"2023-03-25T04:39:53.367398+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:53.367424+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:39:53.367424+0000","last_peered":"2023-03-25T04:39:53.367424+0000","last_clean":"2023-03-25T04:39:53.367424+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:39:53.367424+0000","last_undegraded":"2023-03-25T04:39:53.367424+0000","last_fullsized":"2023-03-25T04:39:53.367424+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117175,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:41:58 2023","interfaces":[{"interface":"back","average":{"1min":0.25800000000000001,"5min":0.26400000000000001,"15min":0.26800000000000002},"min":{"1min":0.17499999999999999,"5min":0.17499999999999999,"15min":0.16},"max":{"1min":0.39100000000000001,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.22900000000000001},{"interface":"front","average":{"1min":0.26300000000000001,"5min":0.27200000000000002,"15min":0.27400000000000002},"min":{"1min":0.20599999999999999,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.35699999999999998,"5min":0.46400000000000002,"15min":0.46400000000000002},"last":0.26400000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051765,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:41:52 2023","interfaces":[{"interface":"back","average":{"1min":0.26500000000000001,"5min":0.27200000000000002,"15min":0.26800000000000002},"min":{"1min":0.17000000000000001,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.39200000000000002,"5min":0.44,"15min":0.44},"last":0.23899999999999999},{"interface":"front","average":{"1min":0.28100000000000003,"5min":0.28599999999999998,"15min":0.28199999999999997},"min":{"1min":0.19400000000000001,"5min":0.19400000000000001,"15min":0.16600000000000001},"max":{"1min":0.41399999999999998,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.254}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:42:18.297+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:42:18.958+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:42:35.512 INFO:teuthology.orchestra.run.smithi070.stdout:
, 2023-03-25T04:42:35.512 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":1013,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":17,"modified":"2023-03-25T04:41:54.262744+0000","services":{}},"progress_events":{}}
, 2023-03-25T04:42:35.525 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:42:52.741 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":548,"stamp":"2023-03-25T04:42:52.273601+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002221"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":40,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:40:23.370056+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:40:23.370056+0000","last_peered":"2023-03-25T04:40:23.370056+0000","last_clean":"2023-03-25T04:40:23.370056+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:40:23.370056+0000","last_undegraded":"2023-03-25T04:40:23.370056+0000","last_fullsized":"2023-03-25T04:40:23.370056+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:40:23.370039+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:40:23.370039+0000","last_peered":"2023-03-25T04:40:23.370039+0000","last_clean":"2023-03-25T04:40:23.370039+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:40:23.370039+0000","last_undegraded":"2023-03-25T04:40:23.370039+0000","last_fullsized":"2023-03-25T04:40:23.370039+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:23.364593+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:39:23.364593+0000","last_peered":"2023-03-25T04:39:23.364593+0000","last_clean":"2023-03-25T04:39:23.364593+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:39:23.364593+0000","last_undegraded":"2023-03-25T04:39:23.364593+0000","last_fullsized":"2023-03-25T04:39:23.364593+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:45.608475+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:39:45.608475+0000","last_peered":"2023-03-25T04:39:45.608475+0000","last_clean":"2023-03-25T04:39:45.608475+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:39:45.608475+0000","last_undegraded":"2023-03-25T04:39:45.608475+0000","last_fullsized":"2023-03-25T04:39:45.608475+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":40,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:53.367398+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:39:53.367398+0000","last_peered":"2023-03-25T04:39:53.367398+0000","last_clean":"2023-03-25T04:39:53.367398+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:39:53.367398+0000","last_undegraded":"2023-03-25T04:39:53.367398+0000","last_fullsized":"2023-03-25T04:39:53.367398+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:53.367424+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:39:53.367424+0000","last_peered":"2023-03-25T04:39:53.367424+0000","last_clean":"2023-03-25T04:39:53.367424+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:39:53.367424+0000","last_undegraded":"2023-03-25T04:39:53.367424+0000","last_fullsized":"2023-03-25T04:39:53.367424+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117182,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:41:58 2023","interfaces":[{"interface":"back","average":{"1min":0.25800000000000001,"5min":0.26400000000000001,"15min":0.26800000000000002},"min":{"1min":0.17499999999999999,"5min":0.17499999999999999,"15min":0.16},"max":{"1min":0.39100000000000001,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.19600000000000001},{"interface":"front","average":{"1min":0.26300000000000001,"5min":0.27200000000000002,"15min":0.27400000000000002},"min":{"1min":0.20599999999999999,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.35699999999999998,"5min":0.46400000000000002,"15min":0.46400000000000002},"last":0.219}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051773,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:41:52 2023","interfaces":[{"interface":"back","average":{"1min":0.26500000000000001,"5min":0.27200000000000002,"15min":0.26800000000000002},"min":{"1min":0.17000000000000001,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.39200000000000002,"5min":0.44,"15min":0.44},"last":0.19},{"interface":"front","average":{"1min":0.28100000000000003,"5min":0.28599999999999998,"15min":0.28199999999999997},"min":{"1min":0.19400000000000001,"5min":0.19400000000000001,"15min":0.16600000000000001},"max":{"1min":0.41399999999999998,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.17599999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:42:52.742 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean

Ignoring malparsed backtrace: 2023-03-25T04:43:09.973 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:43:09.986 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":556,"stamp":"2023-03-25T04:43:08.276556+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002359"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":40,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:40:23.370056+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:40:23.370056+0000","last_peered":"2023-03-25T04:40:23.370056+0000","last_clean":"2023-03-25T04:40:23.370056+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:40:23.370056+0000","last_undegraded":"2023-03-25T04:40:23.370056+0000","last_fullsized":"2023-03-25T04:40:23.370056+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:40:23.370039+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:40:23.370039+0000","last_peered":"2023-03-25T04:40:23.370039+0000","last_clean":"2023-03-25T04:40:23.370039+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:40:23.370039+0000","last_undegraded":"2023-03-25T04:40:23.370039+0000","last_fullsized":"2023-03-25T04:40:23.370039+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:23.364593+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:39:23.364593+0000","last_peered":"2023-03-25T04:39:23.364593+0000","last_clean":"2023-03-25T04:39:23.364593+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:39:23.364593+0000","last_undegraded":"2023-03-25T04:39:23.364593+0000","last_fullsized":"2023-03-25T04:39:23.364593+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:45.608475+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:39:45.608475+0000","last_peered":"2023-03-25T04:39:45.608475+0000","last_clean":"2023-03-25T04:39:45.608475+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:39:45.608475+0000","last_undegraded":"2023-03-25T04:39:45.608475+0000","last_fullsized":"2023-03-25T04:39:45.608475+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":40,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:53.367398+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:39:53.367398+0000","last_peered":"2023-03-25T04:39:53.367398+0000","last_clean":"2023-03-25T04:39:53.367398+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:39:53.367398+0000","last_undegraded":"2023-03-25T04:39:53.367398+0000","last_fullsized":"2023-03-25T04:39:53.367398+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:53.367424+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:39:53.367424+0000","last_peered":"2023-03-25T04:39:53.367424+0000","last_clean":"2023-03-25T04:39:53.367424+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:39:53.367424+0000","last_undegraded":"2023-03-25T04:39:53.367424+0000","last_fullsized":"2023-03-25T04:39:53.367424+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117185,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:43:00 2023","interfaces":[{"interface":"back","average":{"1min":0.27400000000000002,"5min":0.26700000000000002,"15min":0.26600000000000001},"min":{"1min":0.19600000000000001,"5min":0.17499999999999999,"15min":0.16},"max":{"1min":0.42999999999999999,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.27800000000000002},{"interface":"front","average":{"1min":0.28199999999999997,"5min":0.27500000000000002,"15min":0.27300000000000002},"min":{"1min":0.185,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.47199999999999998,"5min":0.47199999999999998,"15min":0.47199999999999998},"last":0.247}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051776,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:42:54 2023","interfaces":[{"interface":"back","average":{"1min":0.26300000000000001,"5min":0.27000000000000002,"15min":0.26900000000000002},"min":{"1min":0.17399999999999999,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.438,"5min":0.44,"15min":0.44},"last":0.20100000000000001},{"interface":"front","average":{"1min":0.27400000000000002,"5min":0.28199999999999997,"15min":0.28299999999999997},"min":{"1min":0.17599999999999999,"5min":0.17599999999999999,"15min":0.16600000000000001},"max":{"1min":0.41599999999999998,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.23499999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:43:25.042+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:43:25.411+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:43:26.013+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:43:47.937+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:43:48.534+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:43:48.889+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:44:09.318 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:44:09.331 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":586,"stamp":"2023-03-25T04:44:08.288933+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002697"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":40,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:40:23.370056+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:40:23.370056+0000","last_peered":"2023-03-25T04:40:23.370056+0000","last_clean":"2023-03-25T04:40:23.370056+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:40:23.370056+0000","last_undegraded":"2023-03-25T04:40:23.370056+0000","last_fullsized":"2023-03-25T04:40:23.370056+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:40:23.370039+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:40:23.370039+0000","last_peered":"2023-03-25T04:40:23.370039+0000","last_clean":"2023-03-25T04:40:23.370039+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:40:23.370039+0000","last_undegraded":"2023-03-25T04:40:23.370039+0000","last_fullsized":"2023-03-25T04:40:23.370039+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:23.375106+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:41:23.375106+0000","last_peered":"2023-03-25T04:41:23.375106+0000","last_clean":"2023-03-25T04:41:23.375106+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:41:23.375106+0000","last_undegraded":"2023-03-25T04:41:23.375106+0000","last_fullsized":"2023-03-25T04:41:23.375106+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:50.618930+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:41:50.618930+0000","last_peered":"2023-03-25T04:41:50.618930+0000","last_clean":"2023-03-25T04:41:50.618930+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:41:50.618930+0000","last_undegraded":"2023-03-25T04:41:50.618930+0000","last_fullsized":"2023-03-25T04:41:50.618930+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":41,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:58.377821+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:41:58.377821+0000","last_peered":"2023-03-25T04:41:58.377821+0000","last_clean":"2023-03-25T04:41:58.377821+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:41:58.377821+0000","last_undegraded":"2023-03-25T04:41:58.377821+0000","last_fullsized":"2023-03-25T04:41:58.377821+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:58.377836+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:41:58.377836+0000","last_peered":"2023-03-25T04:41:58.377836+0000","last_clean":"2023-03-25T04:41:58.377836+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:41:58.377836+0000","last_undegraded":"2023-03-25T04:41:58.377836+0000","last_fullsized":"2023-03-25T04:41:58.377836+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117197,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:44:01 2023","interfaces":[{"interface":"back","average":{"1min":0.27200000000000002,"5min":0.27000000000000002,"15min":0.26400000000000001},"min":{"1min":0.20399999999999999,"5min":0.17499999999999999,"15min":0.16},"max":{"1min":0.38400000000000001,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.245},{"interface":"front","average":{"1min":0.27600000000000002,"5min":0.27800000000000002,"15min":0.27100000000000002},"min":{"1min":0.23100000000000001,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.40400000000000003,"5min":0.47199999999999998,"15min":0.47199999999999998},"last":0.27500000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051788,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:43:55 2023","interfaces":[{"interface":"back","average":{"1min":0.26100000000000001,"5min":0.26700000000000002,"15min":0.27000000000000002},"min":{"1min":0.17799999999999999,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.374,"5min":0.44,"15min":0.44},"last":0.30299999999999999},{"interface":"front","average":{"1min":0.28299999999999997,"5min":0.28100000000000003,"15min":0.28399999999999997},"min":{"1min":0.20899999999999999,"5min":0.17599999999999999,"15min":0.16600000000000001},"max":{"1min":0.40100000000000002,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.27800000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:44:28.923 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":1126,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":19,"modified":"2023-03-25T04:43:20.278959+0000","services":{}},"progress_events":{}}
, 2023-03-25T04:44:28.936 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now
, 2023-03-25T04:44:28.937 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace: 2023-03-25T04:44:48.786 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:44:48.800 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":606,"stamp":"2023-03-25T04:44:48.296805+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002182"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":41,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:42:34.382196+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:42:34.382196+0000","last_peered":"2023-03-25T04:42:34.382196+0000","last_clean":"2023-03-25T04:42:34.382196+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:42:34.382196+0000","last_undegraded":"2023-03-25T04:42:34.382196+0000","last_fullsized":"2023-03-25T04:42:34.382196+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:42:29.381744+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:42:29.381744+0000","last_peered":"2023-03-25T04:42:29.381744+0000","last_clean":"2023-03-25T04:42:29.381744+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:42:29.381744+0000","last_undegraded":"2023-03-25T04:42:29.381744+0000","last_fullsized":"2023-03-25T04:42:29.381744+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:23.375106+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:41:23.375106+0000","last_peered":"2023-03-25T04:41:23.375106+0000","last_clean":"2023-03-25T04:41:23.375106+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:41:23.375106+0000","last_undegraded":"2023-03-25T04:41:23.375106+0000","last_fullsized":"2023-03-25T04:41:23.375106+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:50.618930+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:41:50.618930+0000","last_peered":"2023-03-25T04:41:50.618930+0000","last_clean":"2023-03-25T04:41:50.618930+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:41:50.618930+0000","last_undegraded":"2023-03-25T04:41:50.618930+0000","last_fullsized":"2023-03-25T04:41:50.618930+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":41,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:58.377821+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:41:58.377821+0000","last_peered":"2023-03-25T04:41:58.377821+0000","last_clean":"2023-03-25T04:41:58.377821+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:41:58.377821+0000","last_undegraded":"2023-03-25T04:41:58.377821+0000","last_fullsized":"2023-03-25T04:41:58.377821+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:58.377836+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:41:58.377836+0000","last_peered":"2023-03-25T04:41:58.377836+0000","last_clean":"2023-03-25T04:41:58.377836+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:41:58.377836+0000","last_undegraded":"2023-03-25T04:41:58.377836+0000","last_fullsized":"2023-03-25T04:41:58.377836+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117206,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:44:01 2023","interfaces":[{"interface":"back","average":{"1min":0.27200000000000002,"5min":0.27000000000000002,"15min":0.26400000000000001},"min":{"1min":0.20399999999999999,"5min":0.17499999999999999,"15min":0.16},"max":{"1min":0.38400000000000001,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.216},{"interface":"front","average":{"1min":0.27600000000000002,"5min":0.27800000000000002,"15min":0.27100000000000002},"min":{"1min":0.23100000000000001,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.40400000000000003,"5min":0.47199999999999998,"15min":0.47199999999999998},"last":0.23999999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051796,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:43:55 2023","interfaces":[{"interface":"back","average":{"1min":0.26100000000000001,"5min":0.26700000000000002,"15min":0.27000000000000002},"min":{"1min":0.17799999999999999,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.374,"5min":0.44,"15min":0.44},"last":0.25600000000000001},{"interface":"front","average":{"1min":0.28299999999999997,"5min":0.28100000000000003,"15min":0.28399999999999997},"min":{"1min":0.20899999999999999,"5min":0.17599999999999999,"15min":0.16600000000000001},"max":{"1min":0.40100000000000002,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.28000000000000003}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:45:05.762 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:45:06.073 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all

Ignoring malparsed backtrace: 2023-03-25T04:45:25.658 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":624,"stamp":"2023-03-25T04:45:24.304294+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002366"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":41,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:42:34.382196+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:42:34.382196+0000","last_peered":"2023-03-25T04:42:34.382196+0000","last_clean":"2023-03-25T04:42:34.382196+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:42:34.382196+0000","last_undegraded":"2023-03-25T04:42:34.382196+0000","last_fullsized":"2023-03-25T04:42:34.382196+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:42:29.381744+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:42:29.381744+0000","last_peered":"2023-03-25T04:42:29.381744+0000","last_clean":"2023-03-25T04:42:29.381744+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:42:29.381744+0000","last_undegraded":"2023-03-25T04:42:29.381744+0000","last_fullsized":"2023-03-25T04:42:29.381744+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:23.375106+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:41:23.375106+0000","last_peered":"2023-03-25T04:41:23.375106+0000","last_clean":"2023-03-25T04:41:23.375106+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:41:23.375106+0000","last_undegraded":"2023-03-25T04:41:23.375106+0000","last_fullsized":"2023-03-25T04:41:23.375106+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:50.618930+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:41:50.618930+0000","last_peered":"2023-03-25T04:41:50.618930+0000","last_clean":"2023-03-25T04:41:50.618930+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:41:50.618930+0000","last_undegraded":"2023-03-25T04:41:50.618930+0000","last_fullsized":"2023-03-25T04:41:50.618930+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":41,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:58.377821+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:41:58.377821+0000","last_peered":"2023-03-25T04:41:58.377821+0000","last_clean":"2023-03-25T04:41:58.377821+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:41:58.377821+0000","last_undegraded":"2023-03-25T04:41:58.377821+0000","last_fullsized":"2023-03-25T04:41:58.377821+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:58.377836+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:41:58.377836+0000","last_peered":"2023-03-25T04:41:58.377836+0000","last_clean":"2023-03-25T04:41:58.377836+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:41:58.377836+0000","last_undegraded":"2023-03-25T04:41:58.377836+0000","last_fullsized":"2023-03-25T04:41:58.377836+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117213,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:45:01 2023","interfaces":[{"interface":"back","average":{"1min":0.251,"5min":0.26800000000000002,"15min":0.26400000000000001},"min":{"1min":0.183,"5min":0.17499999999999999,"15min":0.16},"max":{"1min":0.41499999999999998,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.26900000000000002},{"interface":"front","average":{"1min":0.28000000000000003,"5min":0.27900000000000003,"15min":0.27100000000000002},"min":{"1min":0.20999999999999999,"5min":0.185,"15min":0.17499999999999999},"max":{"1min":0.44400000000000001,"5min":0.47199999999999998,"15min":0.47199999999999998},"last":0.27900000000000003}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051803,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:44:55 2023","interfaces":[{"interface":"back","average":{"1min":0.252,"5min":0.26500000000000001,"15min":0.26900000000000002},"min":{"1min":0.183,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.39700000000000002,"5min":0.44,"15min":0.44},"last":0.20699999999999999},{"interface":"front","average":{"1min":0.27000000000000002,"5min":0.28100000000000003,"15min":0.28299999999999997},"min":{"1min":0.20300000000000001,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":0.42099999999999999,"5min":0.47599999999999998,"15min":0.48599999999999999},"last":0.23599999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:45:25.659 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:45:25.969 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:45:42.735+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:45:42.744+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:45:42.897 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace: 2023-03-25T04:46:00.183 INFO:teuthology.orchestra.run.smithi070.stdout:
, 2023-03-25T04:46:00.183 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":1217,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":20,"modified":"2023-03-25T04:45:42.307700+0000","services":{"mgr":{"daemons":{"summary":"","x":{"start_epoch":0,"start_stamp":"0.000000","gid":0,"addr":"(unrecognized address family 0)/0","metadata":{},"task_status":{}}}}}},"progress_events":{}}
, 2023-03-25T04:46:00.197 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now

Ignoring malparsed backtrace: 2023-03-25T04:46:19.910+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:46:20.039 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all

Ignoring malparsed backtrace: 2023-03-25T04:46:39.611 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}
, 2023-03-25T04:46:39.611 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json
, 2023-03-25T04:46:39.613+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:46:58.485+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:46:58.764+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:46:59.519+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:47:14.132 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:47:14.146 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":678,"stamp":"2023-03-25T04:47:12.325044+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002571"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":42,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:44.393426+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:44:44.393426+0000","last_peered":"2023-03-25T04:44:44.393426+0000","last_clean":"2023-03-25T04:44:44.393426+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:44:44.393426+0000","last_undegraded":"2023-03-25T04:44:44.393426+0000","last_fullsized":"2023-03-25T04:44:44.393426+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:39.393044+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:44:39.393044+0000","last_peered":"2023-03-25T04:44:39.393044+0000","last_clean":"2023-03-25T04:44:39.393044+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:44:39.393044+0000","last_undegraded":"2023-03-25T04:44:39.393044+0000","last_fullsized":"2023-03-25T04:44:39.393044+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:43:24.386177+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:43:24.386177+0000","last_peered":"2023-03-25T04:43:24.386177+0000","last_clean":"2023-03-25T04:43:24.386177+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:43:24.386177+0000","last_undegraded":"2023-03-25T04:43:24.386177+0000","last_fullsized":"2023-03-25T04:43:24.386177+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:43:55.629798+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:43:55.629798+0000","last_peered":"2023-03-25T04:43:55.629798+0000","last_clean":"2023-03-25T04:43:55.629798+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:43:55.629798+0000","last_undegraded":"2023-03-25T04:43:55.629798+0000","last_fullsized":"2023-03-25T04:43:55.629798+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":42,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:04.389904+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:44:04.389904+0000","last_peered":"2023-03-25T04:44:04.389904+0000","last_clean":"2023-03-25T04:44:04.389904+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:44:04.389904+0000","last_undegraded":"2023-03-25T04:44:04.389904+0000","last_fullsized":"2023-03-25T04:44:04.389904+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:04.389923+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:44:04.389923+0000","last_peered":"2023-03-25T04:44:04.389923+0000","last_clean":"2023-03-25T04:44:04.389923+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:44:04.389923+0000","last_undegraded":"2023-03-25T04:44:04.389923+0000","last_fullsized":"2023-03-25T04:44:04.389923+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117235,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:47:05 2023","interfaces":[{"interface":"back","average":{"1min":0.27800000000000002,"5min":0.27000000000000002,"15min":0.26300000000000001},"min":{"1min":0.20100000000000001,"5min":0.16900000000000001,"15min":0.16},"max":{"1min":0.39200000000000002,"5min":0.42999999999999999,"15min":0.45600000000000002},"last":0.379},{"interface":"front","average":{"1min":0.28399999999999997,"5min":0.28100000000000003,"15min":0.27200000000000002},"min":{"1min":0.17799999999999999,"5min":0.17799999999999999,"15min":0.17699999999999999},"max":{"1min":0.42599999999999999,"5min":0.47199999999999998,"15min":0.47199999999999998},"last":0.39800000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051825,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:47:01 2023","interfaces":[{"interface":"back","average":{"1min":0.27300000000000002,"5min":0.26100000000000001,"15min":0.27000000000000002},"min":{"1min":0.186,"5min":0.17399999999999999,"15min":0.17000000000000001},"max":{"1min":0.40400000000000003,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.23899999999999999},{"interface":"front","average":{"1min":0.28699999999999998,"5min":0.27500000000000002,"15min":0.28299999999999997},"min":{"1min":0.186,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.48599999999999999},"last":0.19600000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:47:31.198 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json
, 2023-03-25T04:47:31.411+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:47:31.789 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:47:49.509 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":696,"stamp":"2023-03-25T04:47:48.332093+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002697"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":42,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:44.393426+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:44:44.393426+0000","last_peered":"2023-03-25T04:44:44.393426+0000","last_clean":"2023-03-25T04:44:44.393426+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:44:44.393426+0000","last_undegraded":"2023-03-25T04:44:44.393426+0000","last_fullsized":"2023-03-25T04:44:44.393426+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:39.393044+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:44:39.393044+0000","last_peered":"2023-03-25T04:44:39.393044+0000","last_clean":"2023-03-25T04:44:39.393044+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:44:39.393044+0000","last_undegraded":"2023-03-25T04:44:39.393044+0000","last_fullsized":"2023-03-25T04:44:39.393044+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:45:24.396717+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:45:24.396717+0000","last_peered":"2023-03-25T04:45:24.396717+0000","last_clean":"2023-03-25T04:45:24.396717+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:45:24.396717+0000","last_undegraded":"2023-03-25T04:45:24.396717+0000","last_fullsized":"2023-03-25T04:45:24.396717+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:43:55.629798+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:43:55.629798+0000","last_peered":"2023-03-25T04:43:55.629798+0000","last_clean":"2023-03-25T04:43:55.629798+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:43:55.629798+0000","last_undegraded":"2023-03-25T04:43:55.629798+0000","last_fullsized":"2023-03-25T04:43:55.629798+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":42,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:04.389904+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:44:04.389904+0000","last_peered":"2023-03-25T04:44:04.389904+0000","last_clean":"2023-03-25T04:44:04.389904+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:44:04.389904+0000","last_undegraded":"2023-03-25T04:44:04.389904+0000","last_fullsized":"2023-03-25T04:44:04.389904+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:04.389923+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:44:04.389923+0000","last_peered":"2023-03-25T04:44:04.389923+0000","last_clean":"2023-03-25T04:44:04.389923+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:44:04.389923+0000","last_undegraded":"2023-03-25T04:44:04.389923+0000","last_fullsized":"2023-03-25T04:44:04.389923+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117242,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:47:05 2023","interfaces":[{"interface":"back","average":{"1min":0.27800000000000002,"5min":0.27000000000000002,"15min":0.26300000000000001},"min":{"1min":0.20100000000000001,"5min":0.16900000000000001,"15min":0.16},"max":{"1min":0.39200000000000002,"5min":0.42999999999999999,"15min":0.45600000000000002},"last":0.40500000000000003},{"interface":"front","average":{"1min":0.28399999999999997,"5min":0.28100000000000003,"15min":0.27200000000000002},"min":{"1min":0.17799999999999999,"5min":0.17799999999999999,"15min":0.17699999999999999},"max":{"1min":0.42599999999999999,"5min":0.47199999999999998,"15min":0.47199999999999998},"last":0.42399999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051834,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:47:01 2023","interfaces":[{"interface":"back","average":{"1min":0.27300000000000002,"5min":0.26100000000000001,"15min":0.27000000000000002},"min":{"1min":0.186,"5min":0.17399999999999999,"15min":0.17000000000000001},"max":{"1min":0.40400000000000003,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.25600000000000001},{"interface":"front","average":{"1min":0.28699999999999998,"5min":0.27500000000000002,"15min":0.28299999999999997},"min":{"1min":0.186,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.48599999999999999},"last":0.22800000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:47:49.510 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace:     ctx.managers[config['cluster']].wait_for_clean()
,   File "/home/teuthworker/src/github.com_ceph_ceph-c_511015c23eef0684a64171b710e432448df2b7c7/qa/tasks/ceph_manager.py", line 2747, in wait_for_clean
,     assert time.time() - start < timeout, \

Ignoring malparsed backtrace:  ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
,  1: /lib64/libpthread.so.0(+0x12cf0) [0x7f1453c2fcf0]
,  2: __pthread_mutex_lock()

Ignoring malparsed backtrace: 2023-03-25T04:28:05.172 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":142,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":5,"modified":"2023-03-25T04:27:12.092098+0000","services":{}},"progress_events":{}}
, 2023-03-25T04:28:05.185 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now
, 2023-03-25T04:28:05.186 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace: 2023-03-25T04:28:28.360 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":116,"stamp":"2023-03-25T04:28:28.108007+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002578"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.314661+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:27:45.314661+0000","last_peered":"2023-03-25T04:27:45.314661+0000","last_clean":"2023-03-25T04:27:45.314661+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:27:45.314661+0000","last_undegraded":"2023-03-25T04:27:45.314661+0000","last_fullsized":"2023-03-25T04:27:45.314661+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":25,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.310745+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:27:44.310745+0000","last_peered":"2023-03-25T04:27:44.310745+0000","last_clean":"2023-03-25T04:27:44.310745+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:27:44.310745+0000","last_undegraded":"2023-03-25T04:27:44.310745+0000","last_fullsized":"2023-03-25T04:27:44.310745+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":25,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.310650+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:27:44.310650+0000","last_peered":"2023-03-25T04:27:44.310650+0000","last_clean":"2023-03-25T04:27:44.310650+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:27:44.310650+0000","last_undegraded":"2023-03-25T04:27:44.310650+0000","last_fullsized":"2023-03-25T04:27:44.310650+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":26,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.312962+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:27:44.312962+0000","last_peered":"2023-03-25T04:27:44.312962+0000","last_clean":"2023-03-25T04:27:44.312962+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:27:44.312962+0000","last_undegraded":"2023-03-25T04:27:44.312962+0000","last_fullsized":"2023-03-25T04:27:44.312962+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315064+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:27:45.315064+0000","last_peered":"2023-03-25T04:27:45.315064+0000","last_clean":"2023-03-25T04:27:45.315064+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:27:45.315064+0000","last_undegraded":"2023-03-25T04:27:45.315064+0000","last_fullsized":"2023-03-25T04:27:45.315064+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":25,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.310707+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:27:44.310707+0000","last_peered":"2023-03-25T04:27:44.310707+0000","last_clean":"2023-03-25T04:27:44.310707+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:27:44.310707+0000","last_undegraded":"2023-03-25T04:27:44.310707+0000","last_fullsized":"2023-03-25T04:27:44.310707+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117007,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.375}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051597,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.186}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:28:28.362 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean
, 2023-03-25T04:28:28.362 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:28:45.655 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":124,"stamp":"2023-03-25T04:28:44.111265+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002492"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.314661+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:27:45.314661+0000","last_peered":"2023-03-25T04:27:45.314661+0000","last_clean":"2023-03-25T04:27:45.314661+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:27:45.314661+0000","last_undegraded":"2023-03-25T04:27:45.314661+0000","last_fullsized":"2023-03-25T04:27:45.314661+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":25,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.310745+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:27:44.310745+0000","last_peered":"2023-03-25T04:27:44.310745+0000","last_clean":"2023-03-25T04:27:44.310745+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:27:44.310745+0000","last_undegraded":"2023-03-25T04:27:44.310745+0000","last_fullsized":"2023-03-25T04:27:44.310745+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":25,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.310650+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:27:44.310650+0000","last_peered":"2023-03-25T04:27:44.310650+0000","last_clean":"2023-03-25T04:27:44.310650+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:27:44.310650+0000","last_undegraded":"2023-03-25T04:27:44.310650+0000","last_fullsized":"2023-03-25T04:27:44.310650+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":26,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.312962+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:27:44.312962+0000","last_peered":"2023-03-25T04:27:44.312962+0000","last_clean":"2023-03-25T04:27:44.312962+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:27:44.312962+0000","last_undegraded":"2023-03-25T04:27:44.312962+0000","last_fullsized":"2023-03-25T04:27:44.312962+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315064+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:27:45.315064+0000","last_peered":"2023-03-25T04:27:45.315064+0000","last_clean":"2023-03-25T04:27:45.315064+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:27:45.315064+0000","last_undegraded":"2023-03-25T04:27:45.315064+0000","last_fullsized":"2023-03-25T04:27:45.315064+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":25,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.310707+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:27:44.310707+0000","last_peered":"2023-03-25T04:27:44.310707+0000","last_clean":"2023-03-25T04:27:44.310707+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:27:44.310707+0000","last_undegraded":"2023-03-25T04:27:44.310707+0000","last_fullsized":"2023-03-25T04:27:44.310707+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117011,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:28:31 2023","interfaces":[{"interface":"back","average":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.36899999999999999},{"interface":"front","average":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"min":{"1min":0.19,"5min":0.19,"15min":0.19},"max":{"1min":0.46300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.39800000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051601,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:28:24 2023","interfaces":[{"interface":"back","average":{"1min":0.249,"5min":0.249,"15min":0.249},"min":{"1min":0.17699999999999999,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"last":0.254},{"interface":"front","average":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"min":{"1min":0.20200000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"last":0.312}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:28:45.656 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace: 2023-03-25T04:29:05.171 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}
, 2023-03-25T04:29:05.171 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json
, 2023-03-25T04:29:05.561 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:29:22.465 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:29:22.773 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all

Ignoring malparsed backtrace: 2023-03-25T04:29:42.405 INFO:teuthology.orchestra.run.smithi070.stdout:
, 2023-03-25T04:29:42.406 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":240,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":9,"modified":"2023-03-25T04:29:32.120358+0000","services":{"mon":{"daemons":{"summary":"","a":{"start_epoch":0,"start_stamp":"0.000000","gid":0,"addr":"(unrecognized address family 0)/0","metadata":{},"task_status":{}}}}}},"progress_events":{}}
, 2023-03-25T04:29:42.419 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:29:59.627 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":161,"stamp":"2023-03-25T04:29:58.125186+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002398"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315468+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:27:45.315468+0000","last_peered":"2023-03-25T04:27:45.315468+0000","last_clean":"2023-03-25T04:27:45.315468+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:27:45.315468+0000","last_undegraded":"2023-03-25T04:27:45.315468+0000","last_fullsized":"2023-03-25T04:27:45.315468+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312845+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:27:45.312845+0000","last_peered":"2023-03-25T04:27:45.312845+0000","last_clean":"2023-03-25T04:27:45.312845+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:27:45.312845+0000","last_undegraded":"2023-03-25T04:27:45.312845+0000","last_fullsized":"2023-03-25T04:27:45.312845+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312882+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:27:45.312882+0000","last_peered":"2023-03-25T04:27:45.312882+0000","last_clean":"2023-03-25T04:27:45.312882+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:27:45.312882+0000","last_undegraded":"2023-03-25T04:27:45.312882+0000","last_fullsized":"2023-03-25T04:27:45.312882+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.311730+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:27:45.311730+0000","last_peered":"2023-03-25T04:27:45.311730+0000","last_clean":"2023-03-25T04:27:45.311730+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:27:45.311730+0000","last_undegraded":"2023-03-25T04:27:45.311730+0000","last_fullsized":"2023-03-25T04:27:45.311730+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315735+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:27:45.315735+0000","last_peered":"2023-03-25T04:27:45.315735+0000","last_clean":"2023-03-25T04:27:45.315735+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:27:45.315735+0000","last_undegraded":"2023-03-25T04:27:45.315735+0000","last_fullsized":"2023-03-25T04:27:45.315735+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312833+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:27:45.312833+0000","last_peered":"2023-03-25T04:27:45.312833+0000","last_clean":"2023-03-25T04:27:45.312833+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:27:45.312833+0000","last_undegraded":"2023-03-25T04:27:45.312833+0000","last_fullsized":"2023-03-25T04:27:45.312833+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117026,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:29:33 2023","interfaces":[{"interface":"back","average":{"1min":0.26300000000000001,"5min":0.29099999999999998,"15min":0.29599999999999999},"min":{"1min":0.191,"5min":0.191,"15min":0.191},"max":{"1min":0.371,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.20499999999999999},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.29599999999999999,"15min":0.29999999999999999},"min":{"1min":0.21099999999999999,"5min":0.19,"15min":0.19},"max":{"1min":0.39800000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.19500000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051616,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:29:29 2023","interfaces":[{"interface":"back","average":{"1min":0.27200000000000002,"5min":0.253,"15min":0.25},"min":{"1min":0.183,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"last":0.26400000000000001},{"interface":"front","average":{"1min":0.28499999999999998,"5min":0.26800000000000002,"15min":0.26500000000000001},"min":{"1min":0.16600000000000001,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"last":0.29899999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:29:59.629 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean

Ignoring malparsed backtrace: 2023-03-25T04:30:16.889 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:30:16.903 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":170,"stamp":"2023-03-25T04:30:16.128427+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002222"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315468+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:27:45.315468+0000","last_peered":"2023-03-25T04:27:45.315468+0000","last_clean":"2023-03-25T04:27:45.315468+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:27:45.315468+0000","last_undegraded":"2023-03-25T04:27:45.315468+0000","last_fullsized":"2023-03-25T04:27:45.315468+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312845+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:27:45.312845+0000","last_peered":"2023-03-25T04:27:45.312845+0000","last_clean":"2023-03-25T04:27:45.312845+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:27:45.312845+0000","last_undegraded":"2023-03-25T04:27:45.312845+0000","last_fullsized":"2023-03-25T04:27:45.312845+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312882+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:27:45.312882+0000","last_peered":"2023-03-25T04:27:45.312882+0000","last_clean":"2023-03-25T04:27:45.312882+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:27:45.312882+0000","last_undegraded":"2023-03-25T04:27:45.312882+0000","last_fullsized":"2023-03-25T04:27:45.312882+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.311730+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:27:45.311730+0000","last_peered":"2023-03-25T04:27:45.311730+0000","last_clean":"2023-03-25T04:27:45.311730+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:27:45.311730+0000","last_undegraded":"2023-03-25T04:27:45.311730+0000","last_fullsized":"2023-03-25T04:27:45.311730+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315735+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:27:45.315735+0000","last_peered":"2023-03-25T04:27:45.315735+0000","last_clean":"2023-03-25T04:27:45.315735+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:27:45.315735+0000","last_undegraded":"2023-03-25T04:27:45.315735+0000","last_fullsized":"2023-03-25T04:27:45.315735+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312833+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:27:45.312833+0000","last_peered":"2023-03-25T04:27:45.312833+0000","last_clean":"2023-03-25T04:27:45.312833+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:27:45.312833+0000","last_undegraded":"2023-03-25T04:27:45.312833+0000","last_fullsized":"2023-03-25T04:27:45.312833+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117030,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:29:33 2023","interfaces":[{"interface":"back","average":{"1min":0.26300000000000001,"5min":0.29099999999999998,"15min":0.29599999999999999},"min":{"1min":0.191,"5min":0.191,"15min":0.191},"max":{"1min":0.371,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.245},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.29599999999999999,"15min":0.29999999999999999},"min":{"1min":0.21099999999999999,"5min":0.19,"15min":0.19},"max":{"1min":0.39800000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.20899999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051620,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:29:29 2023","interfaces":[{"interface":"back","average":{"1min":0.27200000000000002,"5min":0.253,"15min":0.25},"min":{"1min":0.183,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"last":0.19400000000000001},{"interface":"front","average":{"1min":0.28499999999999998,"5min":0.26800000000000002,"15min":0.26500000000000001},"min":{"1min":0.16600000000000001,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"last":0.23599999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:30:31.984+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:30:32.201+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:30:32.981+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)

Ignoring malparsed backtrace: 2023-03-25T04:30:48.449 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now
, 2023-03-25T04:30:48.450 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:30:48.760 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:31:08.342 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":196,"stamp":"2023-03-25T04:31:08.139018+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002431"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315468+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:27:45.315468+0000","last_peered":"2023-03-25T04:27:45.315468+0000","last_clean":"2023-03-25T04:27:45.315468+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:27:45.315468+0000","last_undegraded":"2023-03-25T04:27:45.315468+0000","last_fullsized":"2023-03-25T04:27:45.315468+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312845+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:27:45.312845+0000","last_peered":"2023-03-25T04:27:45.312845+0000","last_clean":"2023-03-25T04:27:45.312845+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:27:45.312845+0000","last_undegraded":"2023-03-25T04:27:45.312845+0000","last_fullsized":"2023-03-25T04:27:45.312845+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312882+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:27:45.312882+0000","last_peered":"2023-03-25T04:27:45.312882+0000","last_clean":"2023-03-25T04:27:45.312882+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:27:45.312882+0000","last_undegraded":"2023-03-25T04:27:45.312882+0000","last_fullsized":"2023-03-25T04:27:45.312882+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.311730+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:27:45.311730+0000","last_peered":"2023-03-25T04:27:45.311730+0000","last_clean":"2023-03-25T04:27:45.311730+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:27:45.311730+0000","last_undegraded":"2023-03-25T04:27:45.311730+0000","last_fullsized":"2023-03-25T04:27:45.311730+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315735+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:27:45.315735+0000","last_peered":"2023-03-25T04:27:45.315735+0000","last_clean":"2023-03-25T04:27:45.315735+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:27:45.315735+0000","last_undegraded":"2023-03-25T04:27:45.315735+0000","last_fullsized":"2023-03-25T04:27:45.315735+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312833+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:27:45.312833+0000","last_peered":"2023-03-25T04:27:45.312833+0000","last_clean":"2023-03-25T04:27:45.312833+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:27:45.312833+0000","last_undegraded":"2023-03-25T04:27:45.312833+0000","last_fullsized":"2023-03-25T04:27:45.312833+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117040,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:30:37 2023","interfaces":[{"interface":"back","average":{"1min":0.28799999999999998,"5min":0.28899999999999998,"15min":0.29499999999999998},"min":{"1min":0.20200000000000001,"5min":0.191,"15min":0.191},"max":{"1min":0.433,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.29799999999999999},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.28999999999999998,"15min":0.29799999999999999},"min":{"1min":0.17499999999999999,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":0.40300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.25700000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051630,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:30:32 2023","interfaces":[{"interface":"back","average":{"1min":0.25600000000000001,"5min":0.255,"15min":0.251},"min":{"1min":0.19400000000000001,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.35699999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"last":0.22800000000000001},{"interface":"front","average":{"1min":0.28699999999999998,"5min":0.27200000000000002,"15min":0.26600000000000001},"min":{"1min":0.20599999999999999,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.39400000000000002,"5min":0.42899999999999999,"15min":0.42899999999999999},"last":0.24099999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:31:08.343 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace: 2023-03-25T04:31:24.283+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:31:25.192+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:31:25.295+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)

Ignoring malparsed backtrace: 2023-03-25T04:31:40.257 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:31:40.271 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":212,"stamp":"2023-03-25T04:31:40.144732+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002208"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315468+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:27:45.315468+0000","last_peered":"2023-03-25T04:27:45.315468+0000","last_clean":"2023-03-25T04:27:45.315468+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:27:45.315468+0000","last_undegraded":"2023-03-25T04:27:45.315468+0000","last_fullsized":"2023-03-25T04:27:45.315468+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312845+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:27:45.312845+0000","last_peered":"2023-03-25T04:27:45.312845+0000","last_clean":"2023-03-25T04:27:45.312845+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:27:45.312845+0000","last_undegraded":"2023-03-25T04:27:45.312845+0000","last_fullsized":"2023-03-25T04:27:45.312845+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:23.308607+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:29:23.308607+0000","last_peered":"2023-03-25T04:29:23.308607+0000","last_clean":"2023-03-25T04:29:23.308607+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:29:23.308607+0000","last_undegraded":"2023-03-25T04:29:23.308607+0000","last_fullsized":"2023-03-25T04:29:23.308607+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":28,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:20.554710+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:29:20.554710+0000","last_peered":"2023-03-25T04:29:20.554710+0000","last_clean":"2023-03-25T04:29:20.554710+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:29:20.554710+0000","last_undegraded":"2023-03-25T04:29:20.554710+0000","last_fullsized":"2023-03-25T04:29:20.554710+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309216+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:29:28.309216+0000","last_peered":"2023-03-25T04:29:28.309216+0000","last_clean":"2023-03-25T04:29:28.309216+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:29:28.309216+0000","last_undegraded":"2023-03-25T04:29:28.309216+0000","last_fullsized":"2023-03-25T04:29:28.309216+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309233+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:29:28.309233+0000","last_peered":"2023-03-25T04:29:28.309233+0000","last_clean":"2023-03-25T04:29:28.309233+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:29:28.309233+0000","last_undegraded":"2023-03-25T04:29:28.309233+0000","last_fullsized":"2023-03-25T04:29:28.309233+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117046,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:30:37 2023","interfaces":[{"interface":"back","average":{"1min":0.28799999999999998,"5min":0.28899999999999998,"15min":0.29499999999999998},"min":{"1min":0.20200000000000001,"5min":0.191,"15min":0.191},"max":{"1min":0.433,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.216},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.28999999999999998,"15min":0.29799999999999999},"min":{"1min":0.17499999999999999,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":0.40300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.24299999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051637,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:31:35 2023","interfaces":[{"interface":"back","average":{"1min":0.26100000000000001,"5min":0.25700000000000001,"15min":0.251},"min":{"1min":0.19,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.35899999999999999,"5min":0.41599999999999998,"15min":0.41599999999999998},"last":0.221},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.27400000000000002,"15min":0.26700000000000002},"min":{"1min":0.20200000000000001,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.374,"5min":0.42899999999999999,"15min":0.42899999999999999},"last":0.24099999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:32:01.906+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:32:02.388+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:32:02.794 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:32:22.326 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":233,"stamp":"2023-03-25T04:32:22.153289+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002545"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:33.309748+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:29:33.309748+0000","last_peered":"2023-03-25T04:29:33.309748+0000","last_clean":"2023-03-25T04:29:33.309748+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:29:33.309748+0000","last_undegraded":"2023-03-25T04:29:33.309748+0000","last_fullsized":"2023-03-25T04:29:33.309748+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:33.309729+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:29:33.309729+0000","last_peered":"2023-03-25T04:29:33.309729+0000","last_clean":"2023-03-25T04:29:33.309729+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:29:33.309729+0000","last_undegraded":"2023-03-25T04:29:33.309729+0000","last_fullsized":"2023-03-25T04:29:33.309729+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:23.308607+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:29:23.308607+0000","last_peered":"2023-03-25T04:29:23.308607+0000","last_clean":"2023-03-25T04:29:23.308607+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:29:23.308607+0000","last_undegraded":"2023-03-25T04:29:23.308607+0000","last_fullsized":"2023-03-25T04:29:23.308607+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":28,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:20.554710+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:29:20.554710+0000","last_peered":"2023-03-25T04:29:20.554710+0000","last_clean":"2023-03-25T04:29:20.554710+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:29:20.554710+0000","last_undegraded":"2023-03-25T04:29:20.554710+0000","last_fullsized":"2023-03-25T04:29:20.554710+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309216+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:29:28.309216+0000","last_peered":"2023-03-25T04:29:28.309216+0000","last_clean":"2023-03-25T04:29:28.309216+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:29:28.309216+0000","last_undegraded":"2023-03-25T04:29:28.309216+0000","last_fullsized":"2023-03-25T04:29:28.309216+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309233+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:29:28.309233+0000","last_peered":"2023-03-25T04:29:28.309233+0000","last_clean":"2023-03-25T04:29:28.309233+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:29:28.309233+0000","last_undegraded":"2023-03-25T04:29:28.309233+0000","last_fullsized":"2023-03-25T04:29:28.309233+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117055,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:31:42 2023","interfaces":[{"interface":"back","average":{"1min":0.27500000000000002,"5min":0.28399999999999997,"15min":0.29399999999999998},"min":{"1min":0.19800000000000001,"5min":0.191,"15min":0.191},"max":{"1min":0.36899999999999999,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.32300000000000001},{"interface":"front","average":{"1min":0.27900000000000003,"5min":0.28499999999999998,"15min":0.29599999999999999},"min":{"1min":0.216,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":0.40799999999999997,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.27100000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051646,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:31:35 2023","interfaces":[{"interface":"back","average":{"1min":0.26100000000000001,"5min":0.25700000000000001,"15min":0.251},"min":{"1min":0.19,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.35899999999999999,"5min":0.41599999999999998,"15min":0.41599999999999998},"last":0.25600000000000001},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.27400000000000002,"15min":0.26700000000000002},"min":{"1min":0.20200000000000001,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.374,"5min":0.42899999999999999,"15min":0.42899999999999999},"last":0.29199999999999998}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:32:22.327 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean
, 2023-03-25T04:32:22.327 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}

Ignoring malparsed backtrace: 2023-03-25T04:32:39.836 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:32:39.849 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":241,"stamp":"2023-03-25T04:32:38.156312+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002346"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:33.309748+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:29:33.309748+0000","last_peered":"2023-03-25T04:29:33.309748+0000","last_clean":"2023-03-25T04:29:33.309748+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:29:33.309748+0000","last_undegraded":"2023-03-25T04:29:33.309748+0000","last_fullsized":"2023-03-25T04:29:33.309748+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:33.309729+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:29:33.309729+0000","last_peered":"2023-03-25T04:29:33.309729+0000","last_clean":"2023-03-25T04:29:33.309729+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:29:33.309729+0000","last_undegraded":"2023-03-25T04:29:33.309729+0000","last_fullsized":"2023-03-25T04:29:33.309729+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:23.308607+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:29:23.308607+0000","last_peered":"2023-03-25T04:29:23.308607+0000","last_clean":"2023-03-25T04:29:23.308607+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:29:23.308607+0000","last_undegraded":"2023-03-25T04:29:23.308607+0000","last_fullsized":"2023-03-25T04:29:23.308607+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":28,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:20.554710+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:29:20.554710+0000","last_peered":"2023-03-25T04:29:20.554710+0000","last_clean":"2023-03-25T04:29:20.554710+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:29:20.554710+0000","last_undegraded":"2023-03-25T04:29:20.554710+0000","last_fullsized":"2023-03-25T04:29:20.554710+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309216+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:29:28.309216+0000","last_peered":"2023-03-25T04:29:28.309216+0000","last_clean":"2023-03-25T04:29:28.309216+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:29:28.309216+0000","last_undegraded":"2023-03-25T04:29:28.309216+0000","last_fullsized":"2023-03-25T04:29:28.309216+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309233+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:29:28.309233+0000","last_peered":"2023-03-25T04:29:28.309233+0000","last_clean":"2023-03-25T04:29:28.309233+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:29:28.309233+0000","last_undegraded":"2023-03-25T04:29:28.309233+0000","last_fullsized":"2023-03-25T04:29:28.309233+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117058,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:31:42 2023","interfaces":[{"interface":"back","average":{"1min":0.27500000000000002,"5min":0.28399999999999997,"15min":0.29399999999999998},"min":{"1min":0.19800000000000001,"5min":0.191,"15min":0.191},"max":{"1min":0.36899999999999999,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.30599999999999999},{"interface":"front","average":{"1min":0.27900000000000003,"5min":0.28499999999999998,"15min":0.29599999999999999},"min":{"1min":0.216,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":0.40799999999999997,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.32500000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051649,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:31:35 2023","interfaces":[{"interface":"back","average":{"1min":0.26100000000000001,"5min":0.25700000000000001,"15min":0.251},"min":{"1min":0.19,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.35899999999999999,"5min":0.41599999999999998,"15min":0.41599999999999998},"last":0.24199999999999999},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.27400000000000002,"15min":0.26700000000000002},"min":{"1min":0.20200000000000001,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.374,"5min":0.42899999999999999,"15min":0.42899999999999999},"last":0.22600000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:32:54.944+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:32:55.469+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:32:55.927+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)

Ignoring malparsed backtrace: 2023-03-25T04:33:13.958 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean
, 2023-03-25T04:33:13.959 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}
, 2023-03-25T04:33:13.959 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json

Ignoring malparsed backtrace: 2023-03-25T04:33:31.199 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":267,"stamp":"2023-03-25T04:33:30.166660+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002269"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:33.309748+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:29:33.309748+0000","last_peered":"2023-03-25T04:29:33.309748+0000","last_clean":"2023-03-25T04:29:33.309748+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:29:33.309748+0000","last_undegraded":"2023-03-25T04:29:33.309748+0000","last_fullsized":"2023-03-25T04:29:33.309748+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:33.309729+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:29:33.309729+0000","last_peered":"2023-03-25T04:29:33.309729+0000","last_clean":"2023-03-25T04:29:33.309729+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:29:33.309729+0000","last_undegraded":"2023-03-25T04:29:33.309729+0000","last_fullsized":"2023-03-25T04:29:33.309729+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":28,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:31:23.320894+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:31:23.320894+0000","last_peered":"2023-03-25T04:31:23.320894+0000","last_clean":"2023-03-25T04:31:23.320894+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:31:23.320894+0000","last_undegraded":"2023-03-25T04:31:23.320894+0000","last_fullsized":"2023-03-25T04:31:23.320894+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":28,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:20.554710+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:29:20.554710+0000","last_peered":"2023-03-25T04:29:20.554710+0000","last_clean":"2023-03-25T04:29:20.554710+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:29:20.554710+0000","last_undegraded":"2023-03-25T04:29:20.554710+0000","last_fullsized":"2023-03-25T04:29:20.554710+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309216+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:29:28.309216+0000","last_peered":"2023-03-25T04:29:28.309216+0000","last_clean":"2023-03-25T04:29:28.309216+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:29:28.309216+0000","last_undegraded":"2023-03-25T04:29:28.309216+0000","last_fullsized":"2023-03-25T04:29:28.309216+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309233+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:29:28.309233+0000","last_peered":"2023-03-25T04:29:28.309233+0000","last_clean":"2023-03-25T04:29:28.309233+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:29:28.309233+0000","last_undegraded":"2023-03-25T04:29:28.309233+0000","last_fullsized":"2023-03-25T04:29:28.309233+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117068,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:32:46 2023","interfaces":[{"interface":"back","average":{"1min":0.27000000000000002,"5min":0.27900000000000003,"15min":0.29199999999999998},"min":{"1min":0.17899999999999999,"5min":0.17899999999999999,"15min":0.17899999999999999},"max":{"1min":0.35599999999999998,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.28799999999999998},{"interface":"front","average":{"1min":0.26800000000000002,"5min":0.27800000000000002,"15min":0.29399999999999998},"min":{"1min":0.19900000000000001,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":0.377,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.315}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051660,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:32:36 2023","interfaces":[{"interface":"back","average":{"1min":0.248,"5min":0.25700000000000001,"15min":0.251},"min":{"1min":0.17000000000000001,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"last":0.22700000000000001},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.27600000000000002,"15min":0.26800000000000002},"min":{"1min":0.192,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"last":0.20399999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:33:31.201 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:33:31.400+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)

Ignoring malparsed backtrace: 2023-03-25T04:33:47.531+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:33:47.951+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:33:48.505+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)

Ignoring malparsed backtrace: 2023-03-25T04:34:06.104 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json
, 2023-03-25T04:34:06.118+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:34:06.497 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:34:23.399 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:34:23.665+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:34:23.712 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:34:40.195+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:34:40.523+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:34:40.676 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace: 2023-03-25T04:35:03.381+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:35:04.139+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:35:04.391+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)

Ignoring malparsed backtrace: 2023-03-25T04:35:22.738 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean
, 2023-03-25T04:35:22.739 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}
, 2023-03-25T04:35:22.739 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json

Ignoring malparsed backtrace: 2023-03-25T04:35:41.393+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:35:42.352+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:35:42.376+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:35:57.384 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":340,"stamp":"2023-03-25T04:35:56.194992+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.001930"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":36,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:31:43.322894+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:31:43.322894+0000","last_peered":"2023-03-25T04:31:43.322894+0000","last_clean":"2023-03-25T04:31:43.322894+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:31:43.322894+0000","last_undegraded":"2023-03-25T04:31:43.322894+0000","last_fullsized":"2023-03-25T04:31:43.322894+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":28,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:31:43.322876+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:31:43.322876+0000","last_peered":"2023-03-25T04:31:43.322876+0000","last_clean":"2023-03-25T04:31:43.322876+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:31:43.322876+0000","last_undegraded":"2023-03-25T04:31:43.322876+0000","last_fullsized":"2023-03-25T04:31:43.322876+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:23.331998+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:33:23.331998+0000","last_peered":"2023-03-25T04:33:23.331998+0000","last_clean":"2023-03-25T04:33:23.331998+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:33:23.331998+0000","last_undegraded":"2023-03-25T04:33:23.331998+0000","last_fullsized":"2023-03-25T04:33:23.331998+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:30.576723+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:33:30.576723+0000","last_peered":"2023-03-25T04:33:30.576723+0000","last_clean":"2023-03-25T04:33:30.576723+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:33:30.576723+0000","last_undegraded":"2023-03-25T04:33:30.576723+0000","last_fullsized":"2023-03-25T04:33:30.576723+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":37,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:38.333447+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:33:38.333447+0000","last_peered":"2023-03-25T04:33:38.333447+0000","last_clean":"2023-03-25T04:33:38.333447+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:33:38.333447+0000","last_undegraded":"2023-03-25T04:33:38.333447+0000","last_fullsized":"2023-03-25T04:33:38.333447+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:38.333467+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:33:38.333467+0000","last_peered":"2023-03-25T04:33:38.333467+0000","last_clean":"2023-03-25T04:33:38.333467+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:33:38.333467+0000","last_undegraded":"2023-03-25T04:33:38.333467+0000","last_fullsized":"2023-03-25T04:33:38.333467+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117098,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:35:49 2023","interfaces":[{"interface":"back","average":{"1min":0.255,"5min":0.26000000000000001,"15min":0.28199999999999997},"min":{"1min":0.17899999999999999,"5min":0.16,"15min":0.16},"max":{"1min":0.36399999999999999,"5min":0.42599999999999999,"15min":0.45100000000000001},"last":0.26100000000000001},{"interface":"front","average":{"1min":0.26000000000000001,"5min":0.26700000000000002,"15min":0.28599999999999998},"min":{"1min":0.188,"5min":0.17999999999999999,"15min":0.17499999999999999},"max":{"1min":0.39100000000000001,"5min":0.45200000000000001,"15min":0.46300000000000002},"last":0.28399999999999997}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051689,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:35:42 2023","interfaces":[{"interface":"back","average":{"1min":0.29399999999999998,"5min":0.27200000000000002,"15min":0.25800000000000001},"min":{"1min":0.17299999999999999,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.41099999999999998,"5min":0.42699999999999999,"15min":0.42699999999999999},"last":0.20399999999999999},{"interface":"front","average":{"1min":0.30299999999999999,"5min":0.28399999999999997,"15min":0.27300000000000002},"min":{"1min":0.19700000000000001,"5min":0.192,"15min":0.16600000000000001},"max":{"1min":0.43099999999999999,"5min":0.45800000000000002,"15min":0.45800000000000002},"last":0.22}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:35:57.624+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)

Ignoring malparsed backtrace: 2023-03-25T04:36:17.226 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:36:17.239 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":350,"stamp":"2023-03-25T04:36:16.198866+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002148"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":37,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:53.335026+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:33:53.335026+0000","last_peered":"2023-03-25T04:33:53.335026+0000","last_clean":"2023-03-25T04:33:53.335026+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:33:53.335026+0000","last_undegraded":"2023-03-25T04:33:53.335026+0000","last_fullsized":"2023-03-25T04:33:53.335026+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:53.335009+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:33:53.335009+0000","last_peered":"2023-03-25T04:33:53.335009+0000","last_clean":"2023-03-25T04:33:53.335009+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:33:53.335009+0000","last_undegraded":"2023-03-25T04:33:53.335009+0000","last_fullsized":"2023-03-25T04:33:53.335009+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:23.331998+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:33:23.331998+0000","last_peered":"2023-03-25T04:33:23.331998+0000","last_clean":"2023-03-25T04:33:23.331998+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:33:23.331998+0000","last_undegraded":"2023-03-25T04:33:23.331998+0000","last_fullsized":"2023-03-25T04:33:23.331998+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:30.576723+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:33:30.576723+0000","last_peered":"2023-03-25T04:33:30.576723+0000","last_clean":"2023-03-25T04:33:30.576723+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:33:30.576723+0000","last_undegraded":"2023-03-25T04:33:30.576723+0000","last_fullsized":"2023-03-25T04:33:30.576723+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":37,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:38.333447+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:33:38.333447+0000","last_peered":"2023-03-25T04:33:38.333447+0000","last_clean":"2023-03-25T04:33:38.333447+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:33:38.333447+0000","last_undegraded":"2023-03-25T04:33:38.333447+0000","last_fullsized":"2023-03-25T04:33:38.333447+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:38.333467+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:33:38.333467+0000","last_peered":"2023-03-25T04:33:38.333467+0000","last_clean":"2023-03-25T04:33:38.333467+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:33:38.333467+0000","last_undegraded":"2023-03-25T04:33:38.333467+0000","last_fullsized":"2023-03-25T04:33:38.333467+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117102,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:35:49 2023","interfaces":[{"interface":"back","average":{"1min":0.255,"5min":0.26000000000000001,"15min":0.28199999999999997},"min":{"1min":0.17899999999999999,"5min":0.16,"15min":0.16},"max":{"1min":0.36399999999999999,"5min":0.42599999999999999,"15min":0.45100000000000001},"last":0.28399999999999997},{"interface":"front","average":{"1min":0.26000000000000001,"5min":0.26700000000000002,"15min":0.28599999999999998},"min":{"1min":0.188,"5min":0.17999999999999999,"15min":0.17499999999999999},"max":{"1min":0.39100000000000001,"5min":0.45200000000000001,"15min":0.46300000000000002},"last":0.30299999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051693,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:35:42 2023","interfaces":[{"interface":"back","average":{"1min":0.29399999999999998,"5min":0.27200000000000002,"15min":0.25800000000000001},"min":{"1min":0.17299999999999999,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.41099999999999998,"5min":0.42699999999999999,"15min":0.42699999999999999},"last":0.35099999999999998},{"interface":"front","average":{"1min":0.30299999999999999,"5min":0.28399999999999997,"15min":0.27300000000000002},"min":{"1min":0.19700000000000001,"5min":0.192,"15min":0.16600000000000001},"max":{"1min":0.43099999999999999,"5min":0.45800000000000002,"15min":0.45800000000000002},"last":0.32900000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:36:34.556+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:36:34.717+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:36:35.583+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)

Ignoring malparsed backtrace: 2023-03-25T04:36:52.680 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json
, 2023-03-25T04:36:52.701+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:36:53.067 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:37:09.963 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:37:10.273 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all

Ignoring malparsed backtrace: 2023-03-25T04:37:26.856+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:37:27.288 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:37:27.538+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:37:41.903 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":392,"stamp":"2023-03-25T04:37:40.213373+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002012"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":37,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:53.335026+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:33:53.335026+0000","last_peered":"2023-03-25T04:33:53.335026+0000","last_clean":"2023-03-25T04:33:53.335026+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:33:53.335026+0000","last_undegraded":"2023-03-25T04:33:53.335026+0000","last_fullsized":"2023-03-25T04:33:53.335026+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:53.335009+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:33:53.335009+0000","last_peered":"2023-03-25T04:33:53.335009+0000","last_clean":"2023-03-25T04:33:53.335009+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:33:53.335009+0000","last_undegraded":"2023-03-25T04:33:53.335009+0000","last_fullsized":"2023-03-25T04:33:53.335009+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:23.343585+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:35:23.343585+0000","last_peered":"2023-03-25T04:35:23.343585+0000","last_clean":"2023-03-25T04:35:23.343585+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:35:23.343585+0000","last_undegraded":"2023-03-25T04:35:23.343585+0000","last_fullsized":"2023-03-25T04:35:23.343585+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:30.576723+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:33:30.576723+0000","last_peered":"2023-03-25T04:33:30.576723+0000","last_clean":"2023-03-25T04:33:30.576723+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:33:30.576723+0000","last_undegraded":"2023-03-25T04:33:30.576723+0000","last_fullsized":"2023-03-25T04:33:30.576723+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":37,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:38.333447+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:33:38.333447+0000","last_peered":"2023-03-25T04:33:38.333447+0000","last_clean":"2023-03-25T04:33:38.333447+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:33:38.333447+0000","last_undegraded":"2023-03-25T04:33:38.333447+0000","last_fullsized":"2023-03-25T04:33:38.333447+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:38.333467+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:33:38.333467+0000","last_peered":"2023-03-25T04:33:38.333467+0000","last_clean":"2023-03-25T04:33:38.333467+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:33:38.333467+0000","last_undegraded":"2023-03-25T04:33:38.333467+0000","last_fullsized":"2023-03-25T04:33:38.333467+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117119,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:36:50 2023","interfaces":[{"interface":"back","average":{"1min":0.254,"5min":0.25600000000000001,"15min":0.27900000000000003},"min":{"1min":0.20499999999999999,"5min":0.16,"15min":0.16},"max":{"1min":0.371,"5min":0.42599999999999999,"15min":0.45100000000000001},"last":0.23000000000000001},{"interface":"front","average":{"1min":0.26800000000000002,"5min":0.26500000000000001,"15min":0.28399999999999997},"min":{"1min":0.19700000000000001,"5min":0.17999999999999999,"15min":0.17499999999999999},"max":{"1min":0.34699999999999998,"5min":0.45200000000000001,"15min":0.46300000000000002},"last":0.20899999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051710,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:36:44 2023","interfaces":[{"interface":"back","average":{"1min":0.28399999999999997,"5min":0.27700000000000002,"15min":0.26100000000000001},"min":{"1min":0.192,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.40899999999999997,"5min":0.42699999999999999,"15min":0.42699999999999999},"last":0.39700000000000002},{"interface":"front","average":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.27500000000000002},"min":{"1min":0.182,"5min":0.182,"15min":0.16600000000000001},"max":{"1min":0.439,"5min":0.45800000000000002,"15min":0.45800000000000002},"last":0.439}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:37:42.400+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)

Ignoring malparsed backtrace: 2023-03-25T04:38:04.766+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:38:04.922 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:38:05.246 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:38:19.605 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":411,"stamp":"2023-03-25T04:38:18.220966+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002368"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":38,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:36:03.347070+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:36:03.347070+0000","last_peered":"2023-03-25T04:36:03.347070+0000","last_clean":"2023-03-25T04:36:03.347070+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:36:03.347070+0000","last_undegraded":"2023-03-25T04:36:03.347070+0000","last_fullsized":"2023-03-25T04:36:03.347070+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:36:03.347025+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:36:03.347025+0000","last_peered":"2023-03-25T04:36:03.347025+0000","last_clean":"2023-03-25T04:36:03.347025+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:36:03.347025+0000","last_undegraded":"2023-03-25T04:36:03.347025+0000","last_fullsized":"2023-03-25T04:36:03.347025+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:23.343585+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:35:23.343585+0000","last_peered":"2023-03-25T04:35:23.343585+0000","last_clean":"2023-03-25T04:35:23.343585+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:35:23.343585+0000","last_undegraded":"2023-03-25T04:35:23.343585+0000","last_fullsized":"2023-03-25T04:35:23.343585+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:35.587025+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:35:35.587025+0000","last_peered":"2023-03-25T04:35:35.587025+0000","last_clean":"2023-03-25T04:35:35.587025+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:35:35.587025+0000","last_undegraded":"2023-03-25T04:35:35.587025+0000","last_fullsized":"2023-03-25T04:35:35.587025+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":38,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:43.345308+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:35:43.345308+0000","last_peered":"2023-03-25T04:35:43.345308+0000","last_clean":"2023-03-25T04:35:43.345308+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:35:43.345308+0000","last_undegraded":"2023-03-25T04:35:43.345308+0000","last_fullsized":"2023-03-25T04:35:43.345308+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:43.345324+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:35:43.345324+0000","last_peered":"2023-03-25T04:35:43.345324+0000","last_clean":"2023-03-25T04:35:43.345324+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:35:43.345324+0000","last_undegraded":"2023-03-25T04:35:43.345324+0000","last_fullsized":"2023-03-25T04:35:43.345324+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117127,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:37:53 2023","interfaces":[{"interface":"back","average":{"1min":0.25800000000000001,"5min":0.253,"15min":0.27700000000000002},"min":{"1min":0.184,"5min":0.16,"15min":0.16},"max":{"1min":0.44600000000000001,"5min":0.44600000000000001,"15min":0.45100000000000001},"last":0.35199999999999998},{"interface":"front","average":{"1min":0.26900000000000002,"5min":0.26500000000000001,"15min":0.28199999999999997},"min":{"1min":0.192,"5min":0.17999999999999999,"15min":0.17499999999999999},"max":{"1min":0.46400000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"last":0.33200000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051717,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:37:44 2023","interfaces":[{"interface":"back","average":{"1min":0.27300000000000002,"5min":0.28199999999999997,"15min":0.26200000000000001},"min":{"1min":0.17899999999999999,"5min":0.17299999999999999,"15min":0.17000000000000001},"max":{"1min":0.41799999999999998,"5min":0.42099999999999999,"15min":0.42699999999999999},"last":0.28699999999999998},{"interface":"front","average":{"1min":0.29399999999999998,"5min":0.29099999999999998,"15min":0.27700000000000002},"min":{"1min":0.19900000000000001,"5min":0.182,"15min":0.16600000000000001},"max":{"1min":0.441,"5min":0.441,"15min":0.45800000000000002},"last":0.26300000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:38:19.718+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:38:20.315+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)

Ignoring malparsed backtrace: 2023-03-25T04:38:44.283 INFO:teuthology.orchestra.run.smithi070.stdout:
, 2023-03-25T04:38:44.283 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":781,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":16,"modified":"2023-03-25T04:38:40.225321+0000","services":{}},"progress_events":{}}
, 2023-03-25T04:38:44.296 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now

Ignoring malparsed backtrace: 2023-03-25T04:39:02.477+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:39:02.578+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:39:03.524+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:39:20.359 INFO:teuthology.orchestra.run.smithi070.stdout:
, 2023-03-25T04:39:20.359 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":818,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":16,"modified":"2023-03-25T04:38:40.225321+0000","services":{}},"progress_events":{}}
, 2023-03-25T04:39:20.372 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now

Ignoring malparsed backtrace: 2023-03-25T04:39:37.527 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:39:37.541 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":450,"stamp":"2023-03-25T04:39:36.235332+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002086"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":38,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:36:03.347070+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:36:03.347070+0000","last_peered":"2023-03-25T04:36:03.347070+0000","last_clean":"2023-03-25T04:36:03.347070+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:36:03.347070+0000","last_undegraded":"2023-03-25T04:36:03.347070+0000","last_fullsized":"2023-03-25T04:36:03.347070+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:36:03.347025+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:36:03.347025+0000","last_peered":"2023-03-25T04:36:03.347025+0000","last_clean":"2023-03-25T04:36:03.347025+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:36:03.347025+0000","last_undegraded":"2023-03-25T04:36:03.347025+0000","last_fullsized":"2023-03-25T04:36:03.347025+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:23.353778+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:37:23.353778+0000","last_peered":"2023-03-25T04:37:23.353778+0000","last_clean":"2023-03-25T04:37:23.353778+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:37:23.353778+0000","last_undegraded":"2023-03-25T04:37:23.353778+0000","last_fullsized":"2023-03-25T04:37:23.353778+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:35.587025+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:35:35.587025+0000","last_peered":"2023-03-25T04:35:35.587025+0000","last_clean":"2023-03-25T04:35:35.587025+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:35:35.587025+0000","last_undegraded":"2023-03-25T04:35:35.587025+0000","last_fullsized":"2023-03-25T04:35:35.587025+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":38,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:43.345308+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:35:43.345308+0000","last_peered":"2023-03-25T04:35:43.345308+0000","last_clean":"2023-03-25T04:35:43.345308+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:35:43.345308+0000","last_undegraded":"2023-03-25T04:35:43.345308+0000","last_fullsized":"2023-03-25T04:35:43.345308+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:43.345324+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:35:43.345324+0000","last_peered":"2023-03-25T04:35:43.345324+0000","last_clean":"2023-03-25T04:35:43.345324+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:35:43.345324+0000","last_undegraded":"2023-03-25T04:35:43.345324+0000","last_fullsized":"2023-03-25T04:35:43.345324+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117143,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:38:54 2023","interfaces":[{"interface":"back","average":{"1min":0.25800000000000001,"5min":0.25700000000000001,"15min":0.27400000000000002},"min":{"1min":0.192,"5min":0.17399999999999999,"15min":0.16},"max":{"1min":0.39400000000000002,"5min":0.44600000000000001,"15min":0.45100000000000001},"last":0.26500000000000001},{"interface":"front","average":{"1min":0.25900000000000001,"5min":0.26700000000000002,"15min":0.27900000000000003},"min":{"1min":0.19500000000000001,"5min":0.188,"15min":0.17499999999999999},"max":{"1min":0.41499999999999998,"5min":0.46400000000000002,"15min":0.46400000000000002},"last":0.29099999999999998}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051733,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:38:45 2023","interfaces":[{"interface":"back","average":{"1min":0.27300000000000002,"5min":0.28100000000000003,"15min":0.26400000000000001},"min":{"1min":0.19400000000000001,"5min":0.17299999999999999,"15min":0.17000000000000001},"max":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"last":0.27300000000000002},{"interface":"front","average":{"1min":0.28799999999999998,"5min":0.29099999999999998,"15min":0.27800000000000002},"min":{"1min":0.19800000000000001,"5min":0.182,"15min":0.16600000000000001},"max":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"last":0.308}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:39:54.416+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:39:54.455 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:39:54.780 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:40:09.288+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:40:09.389+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:40:10.267+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)

Ignoring malparsed backtrace: 2023-03-25T04:40:28.984 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":476,"stamp":"2023-03-25T04:40:28.245911+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002693"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":39,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:38:13.358968+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:38:13.358968+0000","last_peered":"2023-03-25T04:38:13.358968+0000","last_clean":"2023-03-25T04:38:13.358968+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:38:13.358968+0000","last_undegraded":"2023-03-25T04:38:13.358968+0000","last_fullsized":"2023-03-25T04:38:13.358968+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:38:13.358922+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:38:13.358922+0000","last_peered":"2023-03-25T04:38:13.358922+0000","last_clean":"2023-03-25T04:38:13.358922+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:38:13.358922+0000","last_undegraded":"2023-03-25T04:38:13.358922+0000","last_fullsized":"2023-03-25T04:38:13.358922+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:23.353778+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:37:23.353778+0000","last_peered":"2023-03-25T04:37:23.353778+0000","last_clean":"2023-03-25T04:37:23.353778+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:37:23.353778+0000","last_undegraded":"2023-03-25T04:37:23.353778+0000","last_fullsized":"2023-03-25T04:37:23.353778+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:40.598579+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:37:40.598579+0000","last_peered":"2023-03-25T04:37:40.598579+0000","last_clean":"2023-03-25T04:37:40.598579+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:37:40.598579+0000","last_undegraded":"2023-03-25T04:37:40.598579+0000","last_fullsized":"2023-03-25T04:37:40.598579+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":39,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:48.356344+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:37:48.356344+0000","last_peered":"2023-03-25T04:37:48.356344+0000","last_clean":"2023-03-25T04:37:48.356344+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:37:48.356344+0000","last_undegraded":"2023-03-25T04:37:48.356344+0000","last_fullsized":"2023-03-25T04:37:48.356344+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:48.356365+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:37:48.356365+0000","last_peered":"2023-03-25T04:37:48.356365+0000","last_clean":"2023-03-25T04:37:48.356365+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:37:48.356365+0000","last_undegraded":"2023-03-25T04:37:48.356365+0000","last_fullsized":"2023-03-25T04:37:48.356365+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117153,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:39:54 2023","interfaces":[{"interface":"back","average":{"1min":0.25900000000000001,"5min":0.25600000000000001,"15min":0.27100000000000002},"min":{"1min":0.188,"5min":0.17899999999999999,"15min":0.16},"max":{"1min":0.41999999999999998,"5min":0.44600000000000001,"15min":0.45100000000000001},"last":0.255},{"interface":"front","average":{"1min":0.27500000000000002,"5min":0.26600000000000001,"15min":0.27700000000000002},"min":{"1min":0.17699999999999999,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.45600000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"last":0.26800000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051743,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:39:47 2023","interfaces":[{"interface":"back","average":{"1min":0.26300000000000001,"5min":0.27700000000000002,"15min":0.26500000000000001},"min":{"1min":0.187,"5min":0.17299999999999999,"15min":0.17000000000000001},"max":{"1min":0.433,"5min":0.433,"15min":0.433},"last":0.36599999999999999},{"interface":"front","average":{"1min":0.27100000000000002,"5min":0.28799999999999998,"15min":0.27900000000000003},"min":{"1min":0.20100000000000001,"5min":0.182,"15min":0.16600000000000001},"max":{"1min":0.48599999999999999,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.34999999999999998}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:40:28.986 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean
, 2023-03-25T04:40:28.986 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}

Ignoring malparsed backtrace: 2023-03-25T04:40:47.229+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:40:47.503+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:40:48.251+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)

Ignoring malparsed backtrace: 2023-03-25T04:41:03.267 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:41:03.578 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all

Ignoring malparsed backtrace: 2023-03-25T04:41:21.347 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":502,"stamp":"2023-03-25T04:41:20.256204+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002342"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":39,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:38:13.358968+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:38:13.358968+0000","last_peered":"2023-03-25T04:38:13.358968+0000","last_clean":"2023-03-25T04:38:13.358968+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:38:13.358968+0000","last_undegraded":"2023-03-25T04:38:13.358968+0000","last_fullsized":"2023-03-25T04:38:13.358968+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:38:13.358922+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:38:13.358922+0000","last_peered":"2023-03-25T04:38:13.358922+0000","last_clean":"2023-03-25T04:38:13.358922+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:38:13.358922+0000","last_undegraded":"2023-03-25T04:38:13.358922+0000","last_fullsized":"2023-03-25T04:38:13.358922+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:23.353778+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:37:23.353778+0000","last_peered":"2023-03-25T04:37:23.353778+0000","last_clean":"2023-03-25T04:37:23.353778+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:37:23.353778+0000","last_undegraded":"2023-03-25T04:37:23.353778+0000","last_fullsized":"2023-03-25T04:37:23.353778+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:40.598579+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:37:40.598579+0000","last_peered":"2023-03-25T04:37:40.598579+0000","last_clean":"2023-03-25T04:37:40.598579+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:37:40.598579+0000","last_undegraded":"2023-03-25T04:37:40.598579+0000","last_fullsized":"2023-03-25T04:37:40.598579+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":39,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:48.356344+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:37:48.356344+0000","last_peered":"2023-03-25T04:37:48.356344+0000","last_clean":"2023-03-25T04:37:48.356344+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:37:48.356344+0000","last_undegraded":"2023-03-25T04:37:48.356344+0000","last_fullsized":"2023-03-25T04:37:48.356344+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:48.356365+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:37:48.356365+0000","last_peered":"2023-03-25T04:37:48.356365+0000","last_clean":"2023-03-25T04:37:48.356365+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:37:48.356365+0000","last_undegraded":"2023-03-25T04:37:48.356365+0000","last_fullsized":"2023-03-25T04:37:48.356365+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117163,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:40:56 2023","interfaces":[{"interface":"back","average":{"1min":0.28799999999999998,"5min":0.26300000000000001,"15min":0.27100000000000002},"min":{"1min":0.188,"5min":0.184,"15min":0.16},"max":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.17499999999999999},{"interface":"front","average":{"1min":0.29599999999999999,"5min":0.27300000000000002,"15min":0.27700000000000002},"min":{"1min":0.20599999999999999,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.43099999999999999,"5min":0.46400000000000002,"15min":0.46400000000000002},"last":0.20699999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051754,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:40:52 2023","interfaces":[{"interface":"back","average":{"1min":0.28599999999999998,"5min":0.27500000000000002,"15min":0.26700000000000002},"min":{"1min":0.17899999999999999,"5min":0.17899999999999999,"15min":0.17000000000000001},"max":{"1min":0.44,"5min":0.44,"15min":0.44},"last":0.23699999999999999},{"interface":"front","average":{"1min":0.29799999999999999,"5min":0.28699999999999998,"15min":0.28100000000000003},"min":{"1min":0.20300000000000001,"5min":0.182,"15min":0.16600000000000001},"max":{"1min":0.47599999999999998,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.28000000000000003}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:41:21.349 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean
, 2023-03-25T04:41:21.349 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}

Ignoring malparsed backtrace: 2023-03-25T04:41:39.867+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:41:40.383+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:41:40.905+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:41:58.219 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json
, 2023-03-25T04:41:58.319+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:41:58.644 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:42:18.193 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":530,"stamp":"2023-03-25T04:42:16.266659+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002356"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":39,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:38:13.358968+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:38:13.358968+0000","last_peered":"2023-03-25T04:38:13.358968+0000","last_clean":"2023-03-25T04:38:13.358968+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:38:13.358968+0000","last_undegraded":"2023-03-25T04:38:13.358968+0000","last_fullsized":"2023-03-25T04:38:13.358968+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:38:13.358922+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:38:13.358922+0000","last_peered":"2023-03-25T04:38:13.358922+0000","last_clean":"2023-03-25T04:38:13.358922+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:38:13.358922+0000","last_undegraded":"2023-03-25T04:38:13.358922+0000","last_fullsized":"2023-03-25T04:38:13.358922+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:23.364593+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:39:23.364593+0000","last_peered":"2023-03-25T04:39:23.364593+0000","last_clean":"2023-03-25T04:39:23.364593+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:39:23.364593+0000","last_undegraded":"2023-03-25T04:39:23.364593+0000","last_fullsized":"2023-03-25T04:39:23.364593+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:45.608475+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:39:45.608475+0000","last_peered":"2023-03-25T04:39:45.608475+0000","last_clean":"2023-03-25T04:39:45.608475+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:39:45.608475+0000","last_undegraded":"2023-03-25T04:39:45.608475+0000","last_fullsized":"2023-03-25T04:39:45.608475+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":40,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:53.367398+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:39:53.367398+0000","last_peered":"2023-03-25T04:39:53.367398+0000","last_clean":"2023-03-25T04:39:53.367398+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:39:53.367398+0000","last_undegraded":"2023-03-25T04:39:53.367398+0000","last_fullsized":"2023-03-25T04:39:53.367398+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:53.367424+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:39:53.367424+0000","last_peered":"2023-03-25T04:39:53.367424+0000","last_clean":"2023-03-25T04:39:53.367424+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:39:53.367424+0000","last_undegraded":"2023-03-25T04:39:53.367424+0000","last_fullsized":"2023-03-25T04:39:53.367424+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117175,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:41:58 2023","interfaces":[{"interface":"back","average":{"1min":0.25800000000000001,"5min":0.26400000000000001,"15min":0.26800000000000002},"min":{"1min":0.17499999999999999,"5min":0.17499999999999999,"15min":0.16},"max":{"1min":0.39100000000000001,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.22900000000000001},{"interface":"front","average":{"1min":0.26300000000000001,"5min":0.27200000000000002,"15min":0.27400000000000002},"min":{"1min":0.20599999999999999,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.35699999999999998,"5min":0.46400000000000002,"15min":0.46400000000000002},"last":0.26400000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051765,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:41:52 2023","interfaces":[{"interface":"back","average":{"1min":0.26500000000000001,"5min":0.27200000000000002,"15min":0.26800000000000002},"min":{"1min":0.17000000000000001,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.39200000000000002,"5min":0.44,"15min":0.44},"last":0.23899999999999999},{"interface":"front","average":{"1min":0.28100000000000003,"5min":0.28599999999999998,"15min":0.28199999999999997},"min":{"1min":0.19400000000000001,"5min":0.19400000000000001,"15min":0.16600000000000001},"max":{"1min":0.41399999999999998,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.254}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:42:18.297+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:42:18.958+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:42:35.512 INFO:teuthology.orchestra.run.smithi070.stdout:
, 2023-03-25T04:42:35.512 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":1013,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":17,"modified":"2023-03-25T04:41:54.262744+0000","services":{}},"progress_events":{}}
, 2023-03-25T04:42:35.525 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:42:52.741 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":548,"stamp":"2023-03-25T04:42:52.273601+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002221"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":40,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:40:23.370056+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:40:23.370056+0000","last_peered":"2023-03-25T04:40:23.370056+0000","last_clean":"2023-03-25T04:40:23.370056+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:40:23.370056+0000","last_undegraded":"2023-03-25T04:40:23.370056+0000","last_fullsized":"2023-03-25T04:40:23.370056+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:40:23.370039+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:40:23.370039+0000","last_peered":"2023-03-25T04:40:23.370039+0000","last_clean":"2023-03-25T04:40:23.370039+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:40:23.370039+0000","last_undegraded":"2023-03-25T04:40:23.370039+0000","last_fullsized":"2023-03-25T04:40:23.370039+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:23.364593+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:39:23.364593+0000","last_peered":"2023-03-25T04:39:23.364593+0000","last_clean":"2023-03-25T04:39:23.364593+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:39:23.364593+0000","last_undegraded":"2023-03-25T04:39:23.364593+0000","last_fullsized":"2023-03-25T04:39:23.364593+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:45.608475+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:39:45.608475+0000","last_peered":"2023-03-25T04:39:45.608475+0000","last_clean":"2023-03-25T04:39:45.608475+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:39:45.608475+0000","last_undegraded":"2023-03-25T04:39:45.608475+0000","last_fullsized":"2023-03-25T04:39:45.608475+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":40,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:53.367398+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:39:53.367398+0000","last_peered":"2023-03-25T04:39:53.367398+0000","last_clean":"2023-03-25T04:39:53.367398+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:39:53.367398+0000","last_undegraded":"2023-03-25T04:39:53.367398+0000","last_fullsized":"2023-03-25T04:39:53.367398+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:53.367424+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:39:53.367424+0000","last_peered":"2023-03-25T04:39:53.367424+0000","last_clean":"2023-03-25T04:39:53.367424+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:39:53.367424+0000","last_undegraded":"2023-03-25T04:39:53.367424+0000","last_fullsized":"2023-03-25T04:39:53.367424+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117182,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:41:58 2023","interfaces":[{"interface":"back","average":{"1min":0.25800000000000001,"5min":0.26400000000000001,"15min":0.26800000000000002},"min":{"1min":0.17499999999999999,"5min":0.17499999999999999,"15min":0.16},"max":{"1min":0.39100000000000001,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.19600000000000001},{"interface":"front","average":{"1min":0.26300000000000001,"5min":0.27200000000000002,"15min":0.27400000000000002},"min":{"1min":0.20599999999999999,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.35699999999999998,"5min":0.46400000000000002,"15min":0.46400000000000002},"last":0.219}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051773,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:41:52 2023","interfaces":[{"interface":"back","average":{"1min":0.26500000000000001,"5min":0.27200000000000002,"15min":0.26800000000000002},"min":{"1min":0.17000000000000001,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.39200000000000002,"5min":0.44,"15min":0.44},"last":0.19},{"interface":"front","average":{"1min":0.28100000000000003,"5min":0.28599999999999998,"15min":0.28199999999999997},"min":{"1min":0.19400000000000001,"5min":0.19400000000000001,"15min":0.16600000000000001},"max":{"1min":0.41399999999999998,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.17599999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:42:52.742 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean

Ignoring malparsed backtrace: 2023-03-25T04:43:09.973 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:43:09.986 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":556,"stamp":"2023-03-25T04:43:08.276556+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002359"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":40,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:40:23.370056+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:40:23.370056+0000","last_peered":"2023-03-25T04:40:23.370056+0000","last_clean":"2023-03-25T04:40:23.370056+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:40:23.370056+0000","last_undegraded":"2023-03-25T04:40:23.370056+0000","last_fullsized":"2023-03-25T04:40:23.370056+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:40:23.370039+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:40:23.370039+0000","last_peered":"2023-03-25T04:40:23.370039+0000","last_clean":"2023-03-25T04:40:23.370039+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:40:23.370039+0000","last_undegraded":"2023-03-25T04:40:23.370039+0000","last_fullsized":"2023-03-25T04:40:23.370039+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:23.364593+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:39:23.364593+0000","last_peered":"2023-03-25T04:39:23.364593+0000","last_clean":"2023-03-25T04:39:23.364593+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:39:23.364593+0000","last_undegraded":"2023-03-25T04:39:23.364593+0000","last_fullsized":"2023-03-25T04:39:23.364593+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:45.608475+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:39:45.608475+0000","last_peered":"2023-03-25T04:39:45.608475+0000","last_clean":"2023-03-25T04:39:45.608475+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:39:45.608475+0000","last_undegraded":"2023-03-25T04:39:45.608475+0000","last_fullsized":"2023-03-25T04:39:45.608475+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":40,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:53.367398+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:39:53.367398+0000","last_peered":"2023-03-25T04:39:53.367398+0000","last_clean":"2023-03-25T04:39:53.367398+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:39:53.367398+0000","last_undegraded":"2023-03-25T04:39:53.367398+0000","last_fullsized":"2023-03-25T04:39:53.367398+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:53.367424+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:39:53.367424+0000","last_peered":"2023-03-25T04:39:53.367424+0000","last_clean":"2023-03-25T04:39:53.367424+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:39:53.367424+0000","last_undegraded":"2023-03-25T04:39:53.367424+0000","last_fullsized":"2023-03-25T04:39:53.367424+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117185,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:43:00 2023","interfaces":[{"interface":"back","average":{"1min":0.27400000000000002,"5min":0.26700000000000002,"15min":0.26600000000000001},"min":{"1min":0.19600000000000001,"5min":0.17499999999999999,"15min":0.16},"max":{"1min":0.42999999999999999,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.27800000000000002},{"interface":"front","average":{"1min":0.28199999999999997,"5min":0.27500000000000002,"15min":0.27300000000000002},"min":{"1min":0.185,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.47199999999999998,"5min":0.47199999999999998,"15min":0.47199999999999998},"last":0.247}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051776,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:42:54 2023","interfaces":[{"interface":"back","average":{"1min":0.26300000000000001,"5min":0.27000000000000002,"15min":0.26900000000000002},"min":{"1min":0.17399999999999999,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.438,"5min":0.44,"15min":0.44},"last":0.20100000000000001},{"interface":"front","average":{"1min":0.27400000000000002,"5min":0.28199999999999997,"15min":0.28299999999999997},"min":{"1min":0.17599999999999999,"5min":0.17599999999999999,"15min":0.16600000000000001},"max":{"1min":0.41599999999999998,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.23499999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:43:25.042+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:43:25.411+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:43:26.013+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:43:47.937+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:43:48.534+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:43:48.889+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:44:09.318 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:44:09.331 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":586,"stamp":"2023-03-25T04:44:08.288933+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002697"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":40,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:40:23.370056+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:40:23.370056+0000","last_peered":"2023-03-25T04:40:23.370056+0000","last_clean":"2023-03-25T04:40:23.370056+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:40:23.370056+0000","last_undegraded":"2023-03-25T04:40:23.370056+0000","last_fullsized":"2023-03-25T04:40:23.370056+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:40:23.370039+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:40:23.370039+0000","last_peered":"2023-03-25T04:40:23.370039+0000","last_clean":"2023-03-25T04:40:23.370039+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:40:23.370039+0000","last_undegraded":"2023-03-25T04:40:23.370039+0000","last_fullsized":"2023-03-25T04:40:23.370039+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:23.375106+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:41:23.375106+0000","last_peered":"2023-03-25T04:41:23.375106+0000","last_clean":"2023-03-25T04:41:23.375106+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:41:23.375106+0000","last_undegraded":"2023-03-25T04:41:23.375106+0000","last_fullsized":"2023-03-25T04:41:23.375106+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:50.618930+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:41:50.618930+0000","last_peered":"2023-03-25T04:41:50.618930+0000","last_clean":"2023-03-25T04:41:50.618930+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:41:50.618930+0000","last_undegraded":"2023-03-25T04:41:50.618930+0000","last_fullsized":"2023-03-25T04:41:50.618930+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":41,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:58.377821+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:41:58.377821+0000","last_peered":"2023-03-25T04:41:58.377821+0000","last_clean":"2023-03-25T04:41:58.377821+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:41:58.377821+0000","last_undegraded":"2023-03-25T04:41:58.377821+0000","last_fullsized":"2023-03-25T04:41:58.377821+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:58.377836+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:41:58.377836+0000","last_peered":"2023-03-25T04:41:58.377836+0000","last_clean":"2023-03-25T04:41:58.377836+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:41:58.377836+0000","last_undegraded":"2023-03-25T04:41:58.377836+0000","last_fullsized":"2023-03-25T04:41:58.377836+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117197,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:44:01 2023","interfaces":[{"interface":"back","average":{"1min":0.27200000000000002,"5min":0.27000000000000002,"15min":0.26400000000000001},"min":{"1min":0.20399999999999999,"5min":0.17499999999999999,"15min":0.16},"max":{"1min":0.38400000000000001,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.245},{"interface":"front","average":{"1min":0.27600000000000002,"5min":0.27800000000000002,"15min":0.27100000000000002},"min":{"1min":0.23100000000000001,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.40400000000000003,"5min":0.47199999999999998,"15min":0.47199999999999998},"last":0.27500000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051788,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:43:55 2023","interfaces":[{"interface":"back","average":{"1min":0.26100000000000001,"5min":0.26700000000000002,"15min":0.27000000000000002},"min":{"1min":0.17799999999999999,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.374,"5min":0.44,"15min":0.44},"last":0.30299999999999999},{"interface":"front","average":{"1min":0.28299999999999997,"5min":0.28100000000000003,"15min":0.28399999999999997},"min":{"1min":0.20899999999999999,"5min":0.17599999999999999,"15min":0.16600000000000001},"max":{"1min":0.40100000000000002,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.27800000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:44:28.923 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":1126,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":19,"modified":"2023-03-25T04:43:20.278959+0000","services":{}},"progress_events":{}}
, 2023-03-25T04:44:28.936 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now
, 2023-03-25T04:44:28.937 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace: 2023-03-25T04:44:48.786 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:44:48.800 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":606,"stamp":"2023-03-25T04:44:48.296805+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002182"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":41,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:42:34.382196+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:42:34.382196+0000","last_peered":"2023-03-25T04:42:34.382196+0000","last_clean":"2023-03-25T04:42:34.382196+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:42:34.382196+0000","last_undegraded":"2023-03-25T04:42:34.382196+0000","last_fullsized":"2023-03-25T04:42:34.382196+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:42:29.381744+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:42:29.381744+0000","last_peered":"2023-03-25T04:42:29.381744+0000","last_clean":"2023-03-25T04:42:29.381744+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:42:29.381744+0000","last_undegraded":"2023-03-25T04:42:29.381744+0000","last_fullsized":"2023-03-25T04:42:29.381744+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:23.375106+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:41:23.375106+0000","last_peered":"2023-03-25T04:41:23.375106+0000","last_clean":"2023-03-25T04:41:23.375106+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:41:23.375106+0000","last_undegraded":"2023-03-25T04:41:23.375106+0000","last_fullsized":"2023-03-25T04:41:23.375106+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:50.618930+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:41:50.618930+0000","last_peered":"2023-03-25T04:41:50.618930+0000","last_clean":"2023-03-25T04:41:50.618930+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:41:50.618930+0000","last_undegraded":"2023-03-25T04:41:50.618930+0000","last_fullsized":"2023-03-25T04:41:50.618930+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":41,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:58.377821+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:41:58.377821+0000","last_peered":"2023-03-25T04:41:58.377821+0000","last_clean":"2023-03-25T04:41:58.377821+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:41:58.377821+0000","last_undegraded":"2023-03-25T04:41:58.377821+0000","last_fullsized":"2023-03-25T04:41:58.377821+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:58.377836+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:41:58.377836+0000","last_peered":"2023-03-25T04:41:58.377836+0000","last_clean":"2023-03-25T04:41:58.377836+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:41:58.377836+0000","last_undegraded":"2023-03-25T04:41:58.377836+0000","last_fullsized":"2023-03-25T04:41:58.377836+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117206,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:44:01 2023","interfaces":[{"interface":"back","average":{"1min":0.27200000000000002,"5min":0.27000000000000002,"15min":0.26400000000000001},"min":{"1min":0.20399999999999999,"5min":0.17499999999999999,"15min":0.16},"max":{"1min":0.38400000000000001,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.216},{"interface":"front","average":{"1min":0.27600000000000002,"5min":0.27800000000000002,"15min":0.27100000000000002},"min":{"1min":0.23100000000000001,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.40400000000000003,"5min":0.47199999999999998,"15min":0.47199999999999998},"last":0.23999999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051796,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:43:55 2023","interfaces":[{"interface":"back","average":{"1min":0.26100000000000001,"5min":0.26700000000000002,"15min":0.27000000000000002},"min":{"1min":0.17799999999999999,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.374,"5min":0.44,"15min":0.44},"last":0.25600000000000001},{"interface":"front","average":{"1min":0.28299999999999997,"5min":0.28100000000000003,"15min":0.28399999999999997},"min":{"1min":0.20899999999999999,"5min":0.17599999999999999,"15min":0.16600000000000001},"max":{"1min":0.40100000000000002,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.28000000000000003}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:45:05.762 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:45:06.073 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all

Ignoring malparsed backtrace: 2023-03-25T04:45:25.658 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":624,"stamp":"2023-03-25T04:45:24.304294+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002366"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":41,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:42:34.382196+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:42:34.382196+0000","last_peered":"2023-03-25T04:42:34.382196+0000","last_clean":"2023-03-25T04:42:34.382196+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:42:34.382196+0000","last_undegraded":"2023-03-25T04:42:34.382196+0000","last_fullsized":"2023-03-25T04:42:34.382196+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:42:29.381744+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:42:29.381744+0000","last_peered":"2023-03-25T04:42:29.381744+0000","last_clean":"2023-03-25T04:42:29.381744+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:42:29.381744+0000","last_undegraded":"2023-03-25T04:42:29.381744+0000","last_fullsized":"2023-03-25T04:42:29.381744+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:23.375106+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:41:23.375106+0000","last_peered":"2023-03-25T04:41:23.375106+0000","last_clean":"2023-03-25T04:41:23.375106+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:41:23.375106+0000","last_undegraded":"2023-03-25T04:41:23.375106+0000","last_fullsized":"2023-03-25T04:41:23.375106+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:50.618930+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:41:50.618930+0000","last_peered":"2023-03-25T04:41:50.618930+0000","last_clean":"2023-03-25T04:41:50.618930+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:41:50.618930+0000","last_undegraded":"2023-03-25T04:41:50.618930+0000","last_fullsized":"2023-03-25T04:41:50.618930+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":41,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:58.377821+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:41:58.377821+0000","last_peered":"2023-03-25T04:41:58.377821+0000","last_clean":"2023-03-25T04:41:58.377821+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:41:58.377821+0000","last_undegraded":"2023-03-25T04:41:58.377821+0000","last_fullsized":"2023-03-25T04:41:58.377821+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:58.377836+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:41:58.377836+0000","last_peered":"2023-03-25T04:41:58.377836+0000","last_clean":"2023-03-25T04:41:58.377836+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:41:58.377836+0000","last_undegraded":"2023-03-25T04:41:58.377836+0000","last_fullsized":"2023-03-25T04:41:58.377836+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117213,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:45:01 2023","interfaces":[{"interface":"back","average":{"1min":0.251,"5min":0.26800000000000002,"15min":0.26400000000000001},"min":{"1min":0.183,"5min":0.17499999999999999,"15min":0.16},"max":{"1min":0.41499999999999998,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.26900000000000002},{"interface":"front","average":{"1min":0.28000000000000003,"5min":0.27900000000000003,"15min":0.27100000000000002},"min":{"1min":0.20999999999999999,"5min":0.185,"15min":0.17499999999999999},"max":{"1min":0.44400000000000001,"5min":0.47199999999999998,"15min":0.47199999999999998},"last":0.27900000000000003}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051803,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:44:55 2023","interfaces":[{"interface":"back","average":{"1min":0.252,"5min":0.26500000000000001,"15min":0.26900000000000002},"min":{"1min":0.183,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.39700000000000002,"5min":0.44,"15min":0.44},"last":0.20699999999999999},{"interface":"front","average":{"1min":0.27000000000000002,"5min":0.28100000000000003,"15min":0.28299999999999997},"min":{"1min":0.20300000000000001,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":0.42099999999999999,"5min":0.47599999999999998,"15min":0.48599999999999999},"last":0.23599999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:45:25.659 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:45:25.969 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:45:42.735+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:45:42.744+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:45:42.897 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace: 2023-03-25T04:46:00.183 INFO:teuthology.orchestra.run.smithi070.stdout:
, 2023-03-25T04:46:00.183 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":1217,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":20,"modified":"2023-03-25T04:45:42.307700+0000","services":{"mgr":{"daemons":{"summary":"","x":{"start_epoch":0,"start_stamp":"0.000000","gid":0,"addr":"(unrecognized address family 0)/0","metadata":{},"task_status":{}}}}}},"progress_events":{}}
, 2023-03-25T04:46:00.197 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now

Ignoring malparsed backtrace: 2023-03-25T04:46:19.910+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:46:20.039 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all

Ignoring malparsed backtrace: 2023-03-25T04:46:39.611 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}
, 2023-03-25T04:46:39.611 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json
, 2023-03-25T04:46:39.613+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:46:58.485+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:46:58.764+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:46:59.519+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:47:14.132 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:47:14.146 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":678,"stamp":"2023-03-25T04:47:12.325044+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002571"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":42,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:44.393426+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:44:44.393426+0000","last_peered":"2023-03-25T04:44:44.393426+0000","last_clean":"2023-03-25T04:44:44.393426+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:44:44.393426+0000","last_undegraded":"2023-03-25T04:44:44.393426+0000","last_fullsized":"2023-03-25T04:44:44.393426+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:39.393044+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:44:39.393044+0000","last_peered":"2023-03-25T04:44:39.393044+0000","last_clean":"2023-03-25T04:44:39.393044+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:44:39.393044+0000","last_undegraded":"2023-03-25T04:44:39.393044+0000","last_fullsized":"2023-03-25T04:44:39.393044+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:43:24.386177+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:43:24.386177+0000","last_peered":"2023-03-25T04:43:24.386177+0000","last_clean":"2023-03-25T04:43:24.386177+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:43:24.386177+0000","last_undegraded":"2023-03-25T04:43:24.386177+0000","last_fullsized":"2023-03-25T04:43:24.386177+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:43:55.629798+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:43:55.629798+0000","last_peered":"2023-03-25T04:43:55.629798+0000","last_clean":"2023-03-25T04:43:55.629798+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:43:55.629798+0000","last_undegraded":"2023-03-25T04:43:55.629798+0000","last_fullsized":"2023-03-25T04:43:55.629798+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":42,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:04.389904+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:44:04.389904+0000","last_peered":"2023-03-25T04:44:04.389904+0000","last_clean":"2023-03-25T04:44:04.389904+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:44:04.389904+0000","last_undegraded":"2023-03-25T04:44:04.389904+0000","last_fullsized":"2023-03-25T04:44:04.389904+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:04.389923+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:44:04.389923+0000","last_peered":"2023-03-25T04:44:04.389923+0000","last_clean":"2023-03-25T04:44:04.389923+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:44:04.389923+0000","last_undegraded":"2023-03-25T04:44:04.389923+0000","last_fullsized":"2023-03-25T04:44:04.389923+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117235,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:47:05 2023","interfaces":[{"interface":"back","average":{"1min":0.27800000000000002,"5min":0.27000000000000002,"15min":0.26300000000000001},"min":{"1min":0.20100000000000001,"5min":0.16900000000000001,"15min":0.16},"max":{"1min":0.39200000000000002,"5min":0.42999999999999999,"15min":0.45600000000000002},"last":0.379},{"interface":"front","average":{"1min":0.28399999999999997,"5min":0.28100000000000003,"15min":0.27200000000000002},"min":{"1min":0.17799999999999999,"5min":0.17799999999999999,"15min":0.17699999999999999},"max":{"1min":0.42599999999999999,"5min":0.47199999999999998,"15min":0.47199999999999998},"last":0.39800000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051825,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:47:01 2023","interfaces":[{"interface":"back","average":{"1min":0.27300000000000002,"5min":0.26100000000000001,"15min":0.27000000000000002},"min":{"1min":0.186,"5min":0.17399999999999999,"15min":0.17000000000000001},"max":{"1min":0.40400000000000003,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.23899999999999999},{"interface":"front","average":{"1min":0.28699999999999998,"5min":0.27500000000000002,"15min":0.28299999999999997},"min":{"1min":0.186,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.48599999999999999},"last":0.19600000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:47:31.198 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json
, 2023-03-25T04:47:31.411+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:47:31.789 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:47:49.509 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":696,"stamp":"2023-03-25T04:47:48.332093+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002697"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":42,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:44.393426+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:44:44.393426+0000","last_peered":"2023-03-25T04:44:44.393426+0000","last_clean":"2023-03-25T04:44:44.393426+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:44:44.393426+0000","last_undegraded":"2023-03-25T04:44:44.393426+0000","last_fullsized":"2023-03-25T04:44:44.393426+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:39.393044+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:44:39.393044+0000","last_peered":"2023-03-25T04:44:39.393044+0000","last_clean":"2023-03-25T04:44:39.393044+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:44:39.393044+0000","last_undegraded":"2023-03-25T04:44:39.393044+0000","last_fullsized":"2023-03-25T04:44:39.393044+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:45:24.396717+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:45:24.396717+0000","last_peered":"2023-03-25T04:45:24.396717+0000","last_clean":"2023-03-25T04:45:24.396717+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:45:24.396717+0000","last_undegraded":"2023-03-25T04:45:24.396717+0000","last_fullsized":"2023-03-25T04:45:24.396717+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:43:55.629798+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:43:55.629798+0000","last_peered":"2023-03-25T04:43:55.629798+0000","last_clean":"2023-03-25T04:43:55.629798+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:43:55.629798+0000","last_undegraded":"2023-03-25T04:43:55.629798+0000","last_fullsized":"2023-03-25T04:43:55.629798+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":42,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:04.389904+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:44:04.389904+0000","last_peered":"2023-03-25T04:44:04.389904+0000","last_clean":"2023-03-25T04:44:04.389904+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:44:04.389904+0000","last_undegraded":"2023-03-25T04:44:04.389904+0000","last_fullsized":"2023-03-25T04:44:04.389904+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:04.389923+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:44:04.389923+0000","last_peered":"2023-03-25T04:44:04.389923+0000","last_clean":"2023-03-25T04:44:04.389923+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:44:04.389923+0000","last_undegraded":"2023-03-25T04:44:04.389923+0000","last_fullsized":"2023-03-25T04:44:04.389923+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117242,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:47:05 2023","interfaces":[{"interface":"back","average":{"1min":0.27800000000000002,"5min":0.27000000000000002,"15min":0.26300000000000001},"min":{"1min":0.20100000000000001,"5min":0.16900000000000001,"15min":0.16},"max":{"1min":0.39200000000000002,"5min":0.42999999999999999,"15min":0.45600000000000002},"last":0.40500000000000003},{"interface":"front","average":{"1min":0.28399999999999997,"5min":0.28100000000000003,"15min":0.27200000000000002},"min":{"1min":0.17799999999999999,"5min":0.17799999999999999,"15min":0.17699999999999999},"max":{"1min":0.42599999999999999,"5min":0.47199999999999998,"15min":0.47199999999999998},"last":0.42399999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051834,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:47:01 2023","interfaces":[{"interface":"back","average":{"1min":0.27300000000000002,"5min":0.26100000000000001,"15min":0.27000000000000002},"min":{"1min":0.186,"5min":0.17399999999999999,"15min":0.17000000000000001},"max":{"1min":0.40400000000000003,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.25600000000000001},{"interface":"front","average":{"1min":0.28699999999999998,"5min":0.27500000000000002,"15min":0.28299999999999997},"min":{"1min":0.186,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.48599999999999999},"last":0.22800000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:47:49.510 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace:     ctx.managers[config['cluster']].wait_for_clean()
,   File "/home/teuthworker/src/github.com_ceph_ceph-c_511015c23eef0684a64171b710e432448df2b7c7/qa/tasks/ceph_manager.py", line 2747, in wait_for_clean
,     assert time.time() - start < timeout, \

Ignoring malparsed backtrace:  ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
,  1: /lib64/libpthread.so.0(+0x12cf0) [0x7f1453c2fcf0]
,  2: __pthread_mutex_lock()

Ignoring malparsed backtrace: 2023-03-25T04:28:05.172 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":142,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":5,"modified":"2023-03-25T04:27:12.092098+0000","services":{}},"progress_events":{}}
, 2023-03-25T04:28:05.185 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now
, 2023-03-25T04:28:05.186 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace: 2023-03-25T04:28:28.360 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":116,"stamp":"2023-03-25T04:28:28.108007+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002578"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.314661+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:27:45.314661+0000","last_peered":"2023-03-25T04:27:45.314661+0000","last_clean":"2023-03-25T04:27:45.314661+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:27:45.314661+0000","last_undegraded":"2023-03-25T04:27:45.314661+0000","last_fullsized":"2023-03-25T04:27:45.314661+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":25,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.310745+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:27:44.310745+0000","last_peered":"2023-03-25T04:27:44.310745+0000","last_clean":"2023-03-25T04:27:44.310745+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:27:44.310745+0000","last_undegraded":"2023-03-25T04:27:44.310745+0000","last_fullsized":"2023-03-25T04:27:44.310745+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":25,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.310650+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:27:44.310650+0000","last_peered":"2023-03-25T04:27:44.310650+0000","last_clean":"2023-03-25T04:27:44.310650+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:27:44.310650+0000","last_undegraded":"2023-03-25T04:27:44.310650+0000","last_fullsized":"2023-03-25T04:27:44.310650+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":26,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.312962+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:27:44.312962+0000","last_peered":"2023-03-25T04:27:44.312962+0000","last_clean":"2023-03-25T04:27:44.312962+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:27:44.312962+0000","last_undegraded":"2023-03-25T04:27:44.312962+0000","last_fullsized":"2023-03-25T04:27:44.312962+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315064+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:27:45.315064+0000","last_peered":"2023-03-25T04:27:45.315064+0000","last_clean":"2023-03-25T04:27:45.315064+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:27:45.315064+0000","last_undegraded":"2023-03-25T04:27:45.315064+0000","last_fullsized":"2023-03-25T04:27:45.315064+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":25,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.310707+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:27:44.310707+0000","last_peered":"2023-03-25T04:27:44.310707+0000","last_clean":"2023-03-25T04:27:44.310707+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:27:44.310707+0000","last_undegraded":"2023-03-25T04:27:44.310707+0000","last_fullsized":"2023-03-25T04:27:44.310707+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117007,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.375}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051597,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.186}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:28:28.362 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean
, 2023-03-25T04:28:28.362 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:28:45.655 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":124,"stamp":"2023-03-25T04:28:44.111265+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002492"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.314661+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:27:45.314661+0000","last_peered":"2023-03-25T04:27:45.314661+0000","last_clean":"2023-03-25T04:27:45.314661+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:27:45.314661+0000","last_undegraded":"2023-03-25T04:27:45.314661+0000","last_fullsized":"2023-03-25T04:27:45.314661+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":25,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.310745+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:27:44.310745+0000","last_peered":"2023-03-25T04:27:44.310745+0000","last_clean":"2023-03-25T04:27:44.310745+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:27:44.310745+0000","last_undegraded":"2023-03-25T04:27:44.310745+0000","last_fullsized":"2023-03-25T04:27:44.310745+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":25,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.310650+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:27:44.310650+0000","last_peered":"2023-03-25T04:27:44.310650+0000","last_clean":"2023-03-25T04:27:44.310650+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:27:44.310650+0000","last_undegraded":"2023-03-25T04:27:44.310650+0000","last_fullsized":"2023-03-25T04:27:44.310650+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":26,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.312962+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:27:44.312962+0000","last_peered":"2023-03-25T04:27:44.312962+0000","last_clean":"2023-03-25T04:27:44.312962+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:27:44.312962+0000","last_undegraded":"2023-03-25T04:27:44.312962+0000","last_fullsized":"2023-03-25T04:27:44.312962+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315064+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:27:45.315064+0000","last_peered":"2023-03-25T04:27:45.315064+0000","last_clean":"2023-03-25T04:27:45.315064+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:27:45.315064+0000","last_undegraded":"2023-03-25T04:27:45.315064+0000","last_fullsized":"2023-03-25T04:27:45.315064+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":25,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.310707+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:27:44.310707+0000","last_peered":"2023-03-25T04:27:44.310707+0000","last_clean":"2023-03-25T04:27:44.310707+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:27:44.310707+0000","last_undegraded":"2023-03-25T04:27:44.310707+0000","last_fullsized":"2023-03-25T04:27:44.310707+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117011,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:28:31 2023","interfaces":[{"interface":"back","average":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.36899999999999999},{"interface":"front","average":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"min":{"1min":0.19,"5min":0.19,"15min":0.19},"max":{"1min":0.46300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.39800000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051601,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:28:24 2023","interfaces":[{"interface":"back","average":{"1min":0.249,"5min":0.249,"15min":0.249},"min":{"1min":0.17699999999999999,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"last":0.254},{"interface":"front","average":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"min":{"1min":0.20200000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"last":0.312}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:28:45.656 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace: 2023-03-25T04:29:05.171 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}
, 2023-03-25T04:29:05.171 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json
, 2023-03-25T04:29:05.561 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:29:22.465 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:29:22.773 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all

Ignoring malparsed backtrace: 2023-03-25T04:29:42.405 INFO:teuthology.orchestra.run.smithi070.stdout:
, 2023-03-25T04:29:42.406 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":240,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":9,"modified":"2023-03-25T04:29:32.120358+0000","services":{"mon":{"daemons":{"summary":"","a":{"start_epoch":0,"start_stamp":"0.000000","gid":0,"addr":"(unrecognized address family 0)/0","metadata":{},"task_status":{}}}}}},"progress_events":{}}
, 2023-03-25T04:29:42.419 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:29:59.627 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":161,"stamp":"2023-03-25T04:29:58.125186+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002398"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315468+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:27:45.315468+0000","last_peered":"2023-03-25T04:27:45.315468+0000","last_clean":"2023-03-25T04:27:45.315468+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:27:45.315468+0000","last_undegraded":"2023-03-25T04:27:45.315468+0000","last_fullsized":"2023-03-25T04:27:45.315468+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312845+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:27:45.312845+0000","last_peered":"2023-03-25T04:27:45.312845+0000","last_clean":"2023-03-25T04:27:45.312845+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:27:45.312845+0000","last_undegraded":"2023-03-25T04:27:45.312845+0000","last_fullsized":"2023-03-25T04:27:45.312845+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312882+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:27:45.312882+0000","last_peered":"2023-03-25T04:27:45.312882+0000","last_clean":"2023-03-25T04:27:45.312882+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:27:45.312882+0000","last_undegraded":"2023-03-25T04:27:45.312882+0000","last_fullsized":"2023-03-25T04:27:45.312882+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.311730+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:27:45.311730+0000","last_peered":"2023-03-25T04:27:45.311730+0000","last_clean":"2023-03-25T04:27:45.311730+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:27:45.311730+0000","last_undegraded":"2023-03-25T04:27:45.311730+0000","last_fullsized":"2023-03-25T04:27:45.311730+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315735+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:27:45.315735+0000","last_peered":"2023-03-25T04:27:45.315735+0000","last_clean":"2023-03-25T04:27:45.315735+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:27:45.315735+0000","last_undegraded":"2023-03-25T04:27:45.315735+0000","last_fullsized":"2023-03-25T04:27:45.315735+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312833+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:27:45.312833+0000","last_peered":"2023-03-25T04:27:45.312833+0000","last_clean":"2023-03-25T04:27:45.312833+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:27:45.312833+0000","last_undegraded":"2023-03-25T04:27:45.312833+0000","last_fullsized":"2023-03-25T04:27:45.312833+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117026,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:29:33 2023","interfaces":[{"interface":"back","average":{"1min":0.26300000000000001,"5min":0.29099999999999998,"15min":0.29599999999999999},"min":{"1min":0.191,"5min":0.191,"15min":0.191},"max":{"1min":0.371,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.20499999999999999},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.29599999999999999,"15min":0.29999999999999999},"min":{"1min":0.21099999999999999,"5min":0.19,"15min":0.19},"max":{"1min":0.39800000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.19500000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051616,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:29:29 2023","interfaces":[{"interface":"back","average":{"1min":0.27200000000000002,"5min":0.253,"15min":0.25},"min":{"1min":0.183,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"last":0.26400000000000001},{"interface":"front","average":{"1min":0.28499999999999998,"5min":0.26800000000000002,"15min":0.26500000000000001},"min":{"1min":0.16600000000000001,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"last":0.29899999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:29:59.629 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean

Ignoring malparsed backtrace: 2023-03-25T04:30:16.889 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:30:16.903 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":170,"stamp":"2023-03-25T04:30:16.128427+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002222"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315468+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:27:45.315468+0000","last_peered":"2023-03-25T04:27:45.315468+0000","last_clean":"2023-03-25T04:27:45.315468+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:27:45.315468+0000","last_undegraded":"2023-03-25T04:27:45.315468+0000","last_fullsized":"2023-03-25T04:27:45.315468+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312845+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:27:45.312845+0000","last_peered":"2023-03-25T04:27:45.312845+0000","last_clean":"2023-03-25T04:27:45.312845+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:27:45.312845+0000","last_undegraded":"2023-03-25T04:27:45.312845+0000","last_fullsized":"2023-03-25T04:27:45.312845+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312882+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:27:45.312882+0000","last_peered":"2023-03-25T04:27:45.312882+0000","last_clean":"2023-03-25T04:27:45.312882+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:27:45.312882+0000","last_undegraded":"2023-03-25T04:27:45.312882+0000","last_fullsized":"2023-03-25T04:27:45.312882+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.311730+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:27:45.311730+0000","last_peered":"2023-03-25T04:27:45.311730+0000","last_clean":"2023-03-25T04:27:45.311730+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:27:45.311730+0000","last_undegraded":"2023-03-25T04:27:45.311730+0000","last_fullsized":"2023-03-25T04:27:45.311730+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315735+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:27:45.315735+0000","last_peered":"2023-03-25T04:27:45.315735+0000","last_clean":"2023-03-25T04:27:45.315735+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:27:45.315735+0000","last_undegraded":"2023-03-25T04:27:45.315735+0000","last_fullsized":"2023-03-25T04:27:45.315735+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312833+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:27:45.312833+0000","last_peered":"2023-03-25T04:27:45.312833+0000","last_clean":"2023-03-25T04:27:45.312833+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:27:45.312833+0000","last_undegraded":"2023-03-25T04:27:45.312833+0000","last_fullsized":"2023-03-25T04:27:45.312833+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117030,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:29:33 2023","interfaces":[{"interface":"back","average":{"1min":0.26300000000000001,"5min":0.29099999999999998,"15min":0.29599999999999999},"min":{"1min":0.191,"5min":0.191,"15min":0.191},"max":{"1min":0.371,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.245},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.29599999999999999,"15min":0.29999999999999999},"min":{"1min":0.21099999999999999,"5min":0.19,"15min":0.19},"max":{"1min":0.39800000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.20899999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051620,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:29:29 2023","interfaces":[{"interface":"back","average":{"1min":0.27200000000000002,"5min":0.253,"15min":0.25},"min":{"1min":0.183,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"last":0.19400000000000001},{"interface":"front","average":{"1min":0.28499999999999998,"5min":0.26800000000000002,"15min":0.26500000000000001},"min":{"1min":0.16600000000000001,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"last":0.23599999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:30:31.984+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:30:32.201+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:30:32.981+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)

Ignoring malparsed backtrace: 2023-03-25T04:30:48.449 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now
, 2023-03-25T04:30:48.450 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:30:48.760 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:31:08.342 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":196,"stamp":"2023-03-25T04:31:08.139018+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002431"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315468+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:27:45.315468+0000","last_peered":"2023-03-25T04:27:45.315468+0000","last_clean":"2023-03-25T04:27:45.315468+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:27:45.315468+0000","last_undegraded":"2023-03-25T04:27:45.315468+0000","last_fullsized":"2023-03-25T04:27:45.315468+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312845+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:27:45.312845+0000","last_peered":"2023-03-25T04:27:45.312845+0000","last_clean":"2023-03-25T04:27:45.312845+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:27:45.312845+0000","last_undegraded":"2023-03-25T04:27:45.312845+0000","last_fullsized":"2023-03-25T04:27:45.312845+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312882+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:27:45.312882+0000","last_peered":"2023-03-25T04:27:45.312882+0000","last_clean":"2023-03-25T04:27:45.312882+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:27:45.312882+0000","last_undegraded":"2023-03-25T04:27:45.312882+0000","last_fullsized":"2023-03-25T04:27:45.312882+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.311730+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:27:45.311730+0000","last_peered":"2023-03-25T04:27:45.311730+0000","last_clean":"2023-03-25T04:27:45.311730+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:27:45.311730+0000","last_undegraded":"2023-03-25T04:27:45.311730+0000","last_fullsized":"2023-03-25T04:27:45.311730+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315735+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:27:45.315735+0000","last_peered":"2023-03-25T04:27:45.315735+0000","last_clean":"2023-03-25T04:27:45.315735+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:27:45.315735+0000","last_undegraded":"2023-03-25T04:27:45.315735+0000","last_fullsized":"2023-03-25T04:27:45.315735+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312833+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:27:45.312833+0000","last_peered":"2023-03-25T04:27:45.312833+0000","last_clean":"2023-03-25T04:27:45.312833+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:27:45.312833+0000","last_undegraded":"2023-03-25T04:27:45.312833+0000","last_fullsized":"2023-03-25T04:27:45.312833+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117040,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:30:37 2023","interfaces":[{"interface":"back","average":{"1min":0.28799999999999998,"5min":0.28899999999999998,"15min":0.29499999999999998},"min":{"1min":0.20200000000000001,"5min":0.191,"15min":0.191},"max":{"1min":0.433,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.29799999999999999},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.28999999999999998,"15min":0.29799999999999999},"min":{"1min":0.17499999999999999,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":0.40300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.25700000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051630,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:30:32 2023","interfaces":[{"interface":"back","average":{"1min":0.25600000000000001,"5min":0.255,"15min":0.251},"min":{"1min":0.19400000000000001,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.35699999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"last":0.22800000000000001},{"interface":"front","average":{"1min":0.28699999999999998,"5min":0.27200000000000002,"15min":0.26600000000000001},"min":{"1min":0.20599999999999999,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.39400000000000002,"5min":0.42899999999999999,"15min":0.42899999999999999},"last":0.24099999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:31:08.343 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace: 2023-03-25T04:31:24.283+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:31:25.192+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:31:25.295+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)

Ignoring malparsed backtrace: 2023-03-25T04:31:40.257 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:31:40.271 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":212,"stamp":"2023-03-25T04:31:40.144732+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002208"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315468+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:27:45.315468+0000","last_peered":"2023-03-25T04:27:45.315468+0000","last_clean":"2023-03-25T04:27:45.315468+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:27:45.315468+0000","last_undegraded":"2023-03-25T04:27:45.315468+0000","last_fullsized":"2023-03-25T04:27:45.315468+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312845+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:27:45.312845+0000","last_peered":"2023-03-25T04:27:45.312845+0000","last_clean":"2023-03-25T04:27:45.312845+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:27:45.312845+0000","last_undegraded":"2023-03-25T04:27:45.312845+0000","last_fullsized":"2023-03-25T04:27:45.312845+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:23.308607+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:29:23.308607+0000","last_peered":"2023-03-25T04:29:23.308607+0000","last_clean":"2023-03-25T04:29:23.308607+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:29:23.308607+0000","last_undegraded":"2023-03-25T04:29:23.308607+0000","last_fullsized":"2023-03-25T04:29:23.308607+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":28,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:20.554710+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:29:20.554710+0000","last_peered":"2023-03-25T04:29:20.554710+0000","last_clean":"2023-03-25T04:29:20.554710+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:29:20.554710+0000","last_undegraded":"2023-03-25T04:29:20.554710+0000","last_fullsized":"2023-03-25T04:29:20.554710+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309216+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:29:28.309216+0000","last_peered":"2023-03-25T04:29:28.309216+0000","last_clean":"2023-03-25T04:29:28.309216+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:29:28.309216+0000","last_undegraded":"2023-03-25T04:29:28.309216+0000","last_fullsized":"2023-03-25T04:29:28.309216+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309233+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:29:28.309233+0000","last_peered":"2023-03-25T04:29:28.309233+0000","last_clean":"2023-03-25T04:29:28.309233+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:29:28.309233+0000","last_undegraded":"2023-03-25T04:29:28.309233+0000","last_fullsized":"2023-03-25T04:29:28.309233+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117046,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:30:37 2023","interfaces":[{"interface":"back","average":{"1min":0.28799999999999998,"5min":0.28899999999999998,"15min":0.29499999999999998},"min":{"1min":0.20200000000000001,"5min":0.191,"15min":0.191},"max":{"1min":0.433,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.216},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.28999999999999998,"15min":0.29799999999999999},"min":{"1min":0.17499999999999999,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":0.40300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.24299999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051637,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:31:35 2023","interfaces":[{"interface":"back","average":{"1min":0.26100000000000001,"5min":0.25700000000000001,"15min":0.251},"min":{"1min":0.19,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.35899999999999999,"5min":0.41599999999999998,"15min":0.41599999999999998},"last":0.221},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.27400000000000002,"15min":0.26700000000000002},"min":{"1min":0.20200000000000001,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.374,"5min":0.42899999999999999,"15min":0.42899999999999999},"last":0.24099999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:32:01.906+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:32:02.388+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:32:02.794 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:32:22.326 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":233,"stamp":"2023-03-25T04:32:22.153289+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002545"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:33.309748+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:29:33.309748+0000","last_peered":"2023-03-25T04:29:33.309748+0000","last_clean":"2023-03-25T04:29:33.309748+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:29:33.309748+0000","last_undegraded":"2023-03-25T04:29:33.309748+0000","last_fullsized":"2023-03-25T04:29:33.309748+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:33.309729+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:29:33.309729+0000","last_peered":"2023-03-25T04:29:33.309729+0000","last_clean":"2023-03-25T04:29:33.309729+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:29:33.309729+0000","last_undegraded":"2023-03-25T04:29:33.309729+0000","last_fullsized":"2023-03-25T04:29:33.309729+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:23.308607+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:29:23.308607+0000","last_peered":"2023-03-25T04:29:23.308607+0000","last_clean":"2023-03-25T04:29:23.308607+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:29:23.308607+0000","last_undegraded":"2023-03-25T04:29:23.308607+0000","last_fullsized":"2023-03-25T04:29:23.308607+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":28,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:20.554710+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:29:20.554710+0000","last_peered":"2023-03-25T04:29:20.554710+0000","last_clean":"2023-03-25T04:29:20.554710+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:29:20.554710+0000","last_undegraded":"2023-03-25T04:29:20.554710+0000","last_fullsized":"2023-03-25T04:29:20.554710+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309216+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:29:28.309216+0000","last_peered":"2023-03-25T04:29:28.309216+0000","last_clean":"2023-03-25T04:29:28.309216+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:29:28.309216+0000","last_undegraded":"2023-03-25T04:29:28.309216+0000","last_fullsized":"2023-03-25T04:29:28.309216+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309233+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:29:28.309233+0000","last_peered":"2023-03-25T04:29:28.309233+0000","last_clean":"2023-03-25T04:29:28.309233+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:29:28.309233+0000","last_undegraded":"2023-03-25T04:29:28.309233+0000","last_fullsized":"2023-03-25T04:29:28.309233+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117055,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:31:42 2023","interfaces":[{"interface":"back","average":{"1min":0.27500000000000002,"5min":0.28399999999999997,"15min":0.29399999999999998},"min":{"1min":0.19800000000000001,"5min":0.191,"15min":0.191},"max":{"1min":0.36899999999999999,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.32300000000000001},{"interface":"front","average":{"1min":0.27900000000000003,"5min":0.28499999999999998,"15min":0.29599999999999999},"min":{"1min":0.216,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":0.40799999999999997,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.27100000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051646,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:31:35 2023","interfaces":[{"interface":"back","average":{"1min":0.26100000000000001,"5min":0.25700000000000001,"15min":0.251},"min":{"1min":0.19,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.35899999999999999,"5min":0.41599999999999998,"15min":0.41599999999999998},"last":0.25600000000000001},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.27400000000000002,"15min":0.26700000000000002},"min":{"1min":0.20200000000000001,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.374,"5min":0.42899999999999999,"15min":0.42899999999999999},"last":0.29199999999999998}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:32:22.327 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean
, 2023-03-25T04:32:22.327 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}

Ignoring malparsed backtrace: 2023-03-25T04:32:39.836 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:32:39.849 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":241,"stamp":"2023-03-25T04:32:38.156312+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002346"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:33.309748+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:29:33.309748+0000","last_peered":"2023-03-25T04:29:33.309748+0000","last_clean":"2023-03-25T04:29:33.309748+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:29:33.309748+0000","last_undegraded":"2023-03-25T04:29:33.309748+0000","last_fullsized":"2023-03-25T04:29:33.309748+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:33.309729+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:29:33.309729+0000","last_peered":"2023-03-25T04:29:33.309729+0000","last_clean":"2023-03-25T04:29:33.309729+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:29:33.309729+0000","last_undegraded":"2023-03-25T04:29:33.309729+0000","last_fullsized":"2023-03-25T04:29:33.309729+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:23.308607+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:29:23.308607+0000","last_peered":"2023-03-25T04:29:23.308607+0000","last_clean":"2023-03-25T04:29:23.308607+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:29:23.308607+0000","last_undegraded":"2023-03-25T04:29:23.308607+0000","last_fullsized":"2023-03-25T04:29:23.308607+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":28,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:20.554710+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:29:20.554710+0000","last_peered":"2023-03-25T04:29:20.554710+0000","last_clean":"2023-03-25T04:29:20.554710+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:29:20.554710+0000","last_undegraded":"2023-03-25T04:29:20.554710+0000","last_fullsized":"2023-03-25T04:29:20.554710+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309216+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:29:28.309216+0000","last_peered":"2023-03-25T04:29:28.309216+0000","last_clean":"2023-03-25T04:29:28.309216+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:29:28.309216+0000","last_undegraded":"2023-03-25T04:29:28.309216+0000","last_fullsized":"2023-03-25T04:29:28.309216+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309233+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:29:28.309233+0000","last_peered":"2023-03-25T04:29:28.309233+0000","last_clean":"2023-03-25T04:29:28.309233+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:29:28.309233+0000","last_undegraded":"2023-03-25T04:29:28.309233+0000","last_fullsized":"2023-03-25T04:29:28.309233+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117058,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:31:42 2023","interfaces":[{"interface":"back","average":{"1min":0.27500000000000002,"5min":0.28399999999999997,"15min":0.29399999999999998},"min":{"1min":0.19800000000000001,"5min":0.191,"15min":0.191},"max":{"1min":0.36899999999999999,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.30599999999999999},{"interface":"front","average":{"1min":0.27900000000000003,"5min":0.28499999999999998,"15min":0.29599999999999999},"min":{"1min":0.216,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":0.40799999999999997,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.32500000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051649,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:31:35 2023","interfaces":[{"interface":"back","average":{"1min":0.26100000000000001,"5min":0.25700000000000001,"15min":0.251},"min":{"1min":0.19,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.35899999999999999,"5min":0.41599999999999998,"15min":0.41599999999999998},"last":0.24199999999999999},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.27400000000000002,"15min":0.26700000000000002},"min":{"1min":0.20200000000000001,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.374,"5min":0.42899999999999999,"15min":0.42899999999999999},"last":0.22600000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:32:54.944+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:32:55.469+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:32:55.927+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)

Ignoring malparsed backtrace: 2023-03-25T04:33:13.958 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean
, 2023-03-25T04:33:13.959 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}
, 2023-03-25T04:33:13.959 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json

Ignoring malparsed backtrace: 2023-03-25T04:33:31.199 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":267,"stamp":"2023-03-25T04:33:30.166660+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002269"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:33.309748+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:29:33.309748+0000","last_peered":"2023-03-25T04:29:33.309748+0000","last_clean":"2023-03-25T04:29:33.309748+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:29:33.309748+0000","last_undegraded":"2023-03-25T04:29:33.309748+0000","last_fullsized":"2023-03-25T04:29:33.309748+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:33.309729+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:29:33.309729+0000","last_peered":"2023-03-25T04:29:33.309729+0000","last_clean":"2023-03-25T04:29:33.309729+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:29:33.309729+0000","last_undegraded":"2023-03-25T04:29:33.309729+0000","last_fullsized":"2023-03-25T04:29:33.309729+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":28,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:31:23.320894+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:31:23.320894+0000","last_peered":"2023-03-25T04:31:23.320894+0000","last_clean":"2023-03-25T04:31:23.320894+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:31:23.320894+0000","last_undegraded":"2023-03-25T04:31:23.320894+0000","last_fullsized":"2023-03-25T04:31:23.320894+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":28,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:20.554710+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:29:20.554710+0000","last_peered":"2023-03-25T04:29:20.554710+0000","last_clean":"2023-03-25T04:29:20.554710+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:29:20.554710+0000","last_undegraded":"2023-03-25T04:29:20.554710+0000","last_fullsized":"2023-03-25T04:29:20.554710+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309216+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:29:28.309216+0000","last_peered":"2023-03-25T04:29:28.309216+0000","last_clean":"2023-03-25T04:29:28.309216+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:29:28.309216+0000","last_undegraded":"2023-03-25T04:29:28.309216+0000","last_fullsized":"2023-03-25T04:29:28.309216+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309233+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:29:28.309233+0000","last_peered":"2023-03-25T04:29:28.309233+0000","last_clean":"2023-03-25T04:29:28.309233+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:29:28.309233+0000","last_undegraded":"2023-03-25T04:29:28.309233+0000","last_fullsized":"2023-03-25T04:29:28.309233+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117068,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:32:46 2023","interfaces":[{"interface":"back","average":{"1min":0.27000000000000002,"5min":0.27900000000000003,"15min":0.29199999999999998},"min":{"1min":0.17899999999999999,"5min":0.17899999999999999,"15min":0.17899999999999999},"max":{"1min":0.35599999999999998,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.28799999999999998},{"interface":"front","average":{"1min":0.26800000000000002,"5min":0.27800000000000002,"15min":0.29399999999999998},"min":{"1min":0.19900000000000001,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":0.377,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.315}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051660,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:32:36 2023","interfaces":[{"interface":"back","average":{"1min":0.248,"5min":0.25700000000000001,"15min":0.251},"min":{"1min":0.17000000000000001,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"last":0.22700000000000001},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.27600000000000002,"15min":0.26800000000000002},"min":{"1min":0.192,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"last":0.20399999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:33:31.201 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:33:31.400+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)

Ignoring malparsed backtrace: 2023-03-25T04:33:47.531+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:33:47.951+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:33:48.505+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)

Ignoring malparsed backtrace: 2023-03-25T04:34:06.104 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json
, 2023-03-25T04:34:06.118+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:34:06.497 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:34:23.399 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:34:23.665+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:34:23.712 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:34:40.195+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:34:40.523+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:34:40.676 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace: 2023-03-25T04:35:03.381+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:35:04.139+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:35:04.391+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)

Ignoring malparsed backtrace: 2023-03-25T04:35:22.738 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean
, 2023-03-25T04:35:22.739 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}
, 2023-03-25T04:35:22.739 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json

Ignoring malparsed backtrace: 2023-03-25T04:35:41.393+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:35:42.352+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:35:42.376+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:35:57.384 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":340,"stamp":"2023-03-25T04:35:56.194992+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.001930"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":36,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:31:43.322894+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:31:43.322894+0000","last_peered":"2023-03-25T04:31:43.322894+0000","last_clean":"2023-03-25T04:31:43.322894+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:31:43.322894+0000","last_undegraded":"2023-03-25T04:31:43.322894+0000","last_fullsized":"2023-03-25T04:31:43.322894+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":28,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:31:43.322876+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:31:43.322876+0000","last_peered":"2023-03-25T04:31:43.322876+0000","last_clean":"2023-03-25T04:31:43.322876+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:31:43.322876+0000","last_undegraded":"2023-03-25T04:31:43.322876+0000","last_fullsized":"2023-03-25T04:31:43.322876+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:23.331998+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:33:23.331998+0000","last_peered":"2023-03-25T04:33:23.331998+0000","last_clean":"2023-03-25T04:33:23.331998+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:33:23.331998+0000","last_undegraded":"2023-03-25T04:33:23.331998+0000","last_fullsized":"2023-03-25T04:33:23.331998+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:30.576723+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:33:30.576723+0000","last_peered":"2023-03-25T04:33:30.576723+0000","last_clean":"2023-03-25T04:33:30.576723+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:33:30.576723+0000","last_undegraded":"2023-03-25T04:33:30.576723+0000","last_fullsized":"2023-03-25T04:33:30.576723+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":37,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:38.333447+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:33:38.333447+0000","last_peered":"2023-03-25T04:33:38.333447+0000","last_clean":"2023-03-25T04:33:38.333447+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:33:38.333447+0000","last_undegraded":"2023-03-25T04:33:38.333447+0000","last_fullsized":"2023-03-25T04:33:38.333447+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:38.333467+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:33:38.333467+0000","last_peered":"2023-03-25T04:33:38.333467+0000","last_clean":"2023-03-25T04:33:38.333467+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:33:38.333467+0000","last_undegraded":"2023-03-25T04:33:38.333467+0000","last_fullsized":"2023-03-25T04:33:38.333467+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117098,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:35:49 2023","interfaces":[{"interface":"back","average":{"1min":0.255,"5min":0.26000000000000001,"15min":0.28199999999999997},"min":{"1min":0.17899999999999999,"5min":0.16,"15min":0.16},"max":{"1min":0.36399999999999999,"5min":0.42599999999999999,"15min":0.45100000000000001},"last":0.26100000000000001},{"interface":"front","average":{"1min":0.26000000000000001,"5min":0.26700000000000002,"15min":0.28599999999999998},"min":{"1min":0.188,"5min":0.17999999999999999,"15min":0.17499999999999999},"max":{"1min":0.39100000000000001,"5min":0.45200000000000001,"15min":0.46300000000000002},"last":0.28399999999999997}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051689,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:35:42 2023","interfaces":[{"interface":"back","average":{"1min":0.29399999999999998,"5min":0.27200000000000002,"15min":0.25800000000000001},"min":{"1min":0.17299999999999999,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.41099999999999998,"5min":0.42699999999999999,"15min":0.42699999999999999},"last":0.20399999999999999},{"interface":"front","average":{"1min":0.30299999999999999,"5min":0.28399999999999997,"15min":0.27300000000000002},"min":{"1min":0.19700000000000001,"5min":0.192,"15min":0.16600000000000001},"max":{"1min":0.43099999999999999,"5min":0.45800000000000002,"15min":0.45800000000000002},"last":0.22}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:35:57.624+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)

Ignoring malparsed backtrace: 2023-03-25T04:36:17.226 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:36:17.239 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":350,"stamp":"2023-03-25T04:36:16.198866+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002148"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":37,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:53.335026+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:33:53.335026+0000","last_peered":"2023-03-25T04:33:53.335026+0000","last_clean":"2023-03-25T04:33:53.335026+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:33:53.335026+0000","last_undegraded":"2023-03-25T04:33:53.335026+0000","last_fullsized":"2023-03-25T04:33:53.335026+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:53.335009+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:33:53.335009+0000","last_peered":"2023-03-25T04:33:53.335009+0000","last_clean":"2023-03-25T04:33:53.335009+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:33:53.335009+0000","last_undegraded":"2023-03-25T04:33:53.335009+0000","last_fullsized":"2023-03-25T04:33:53.335009+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:23.331998+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:33:23.331998+0000","last_peered":"2023-03-25T04:33:23.331998+0000","last_clean":"2023-03-25T04:33:23.331998+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:33:23.331998+0000","last_undegraded":"2023-03-25T04:33:23.331998+0000","last_fullsized":"2023-03-25T04:33:23.331998+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:30.576723+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:33:30.576723+0000","last_peered":"2023-03-25T04:33:30.576723+0000","last_clean":"2023-03-25T04:33:30.576723+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:33:30.576723+0000","last_undegraded":"2023-03-25T04:33:30.576723+0000","last_fullsized":"2023-03-25T04:33:30.576723+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":37,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:38.333447+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:33:38.333447+0000","last_peered":"2023-03-25T04:33:38.333447+0000","last_clean":"2023-03-25T04:33:38.333447+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:33:38.333447+0000","last_undegraded":"2023-03-25T04:33:38.333447+0000","last_fullsized":"2023-03-25T04:33:38.333447+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:38.333467+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:33:38.333467+0000","last_peered":"2023-03-25T04:33:38.333467+0000","last_clean":"2023-03-25T04:33:38.333467+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:33:38.333467+0000","last_undegraded":"2023-03-25T04:33:38.333467+0000","last_fullsized":"2023-03-25T04:33:38.333467+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117102,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:35:49 2023","interfaces":[{"interface":"back","average":{"1min":0.255,"5min":0.26000000000000001,"15min":0.28199999999999997},"min":{"1min":0.17899999999999999,"5min":0.16,"15min":0.16},"max":{"1min":0.36399999999999999,"5min":0.42599999999999999,"15min":0.45100000000000001},"last":0.28399999999999997},{"interface":"front","average":{"1min":0.26000000000000001,"5min":0.26700000000000002,"15min":0.28599999999999998},"min":{"1min":0.188,"5min":0.17999999999999999,"15min":0.17499999999999999},"max":{"1min":0.39100000000000001,"5min":0.45200000000000001,"15min":0.46300000000000002},"last":0.30299999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051693,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:35:42 2023","interfaces":[{"interface":"back","average":{"1min":0.29399999999999998,"5min":0.27200000000000002,"15min":0.25800000000000001},"min":{"1min":0.17299999999999999,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.41099999999999998,"5min":0.42699999999999999,"15min":0.42699999999999999},"last":0.35099999999999998},{"interface":"front","average":{"1min":0.30299999999999999,"5min":0.28399999999999997,"15min":0.27300000000000002},"min":{"1min":0.19700000000000001,"5min":0.192,"15min":0.16600000000000001},"max":{"1min":0.43099999999999999,"5min":0.45800000000000002,"15min":0.45800000000000002},"last":0.32900000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:36:34.556+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:36:34.717+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:36:35.583+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)

Ignoring malparsed backtrace: 2023-03-25T04:36:52.680 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json
, 2023-03-25T04:36:52.701+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:36:53.067 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:37:09.963 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:37:10.273 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all

Ignoring malparsed backtrace: 2023-03-25T04:37:26.856+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:37:27.288 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:37:27.538+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:37:41.903 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":392,"stamp":"2023-03-25T04:37:40.213373+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002012"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":37,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:53.335026+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:33:53.335026+0000","last_peered":"2023-03-25T04:33:53.335026+0000","last_clean":"2023-03-25T04:33:53.335026+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:33:53.335026+0000","last_undegraded":"2023-03-25T04:33:53.335026+0000","last_fullsized":"2023-03-25T04:33:53.335026+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:53.335009+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:33:53.335009+0000","last_peered":"2023-03-25T04:33:53.335009+0000","last_clean":"2023-03-25T04:33:53.335009+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:33:53.335009+0000","last_undegraded":"2023-03-25T04:33:53.335009+0000","last_fullsized":"2023-03-25T04:33:53.335009+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:23.343585+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:35:23.343585+0000","last_peered":"2023-03-25T04:35:23.343585+0000","last_clean":"2023-03-25T04:35:23.343585+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:35:23.343585+0000","last_undegraded":"2023-03-25T04:35:23.343585+0000","last_fullsized":"2023-03-25T04:35:23.343585+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:30.576723+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:33:30.576723+0000","last_peered":"2023-03-25T04:33:30.576723+0000","last_clean":"2023-03-25T04:33:30.576723+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:33:30.576723+0000","last_undegraded":"2023-03-25T04:33:30.576723+0000","last_fullsized":"2023-03-25T04:33:30.576723+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":37,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:38.333447+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:33:38.333447+0000","last_peered":"2023-03-25T04:33:38.333447+0000","last_clean":"2023-03-25T04:33:38.333447+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:33:38.333447+0000","last_undegraded":"2023-03-25T04:33:38.333447+0000","last_fullsized":"2023-03-25T04:33:38.333447+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:38.333467+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:33:38.333467+0000","last_peered":"2023-03-25T04:33:38.333467+0000","last_clean":"2023-03-25T04:33:38.333467+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:33:38.333467+0000","last_undegraded":"2023-03-25T04:33:38.333467+0000","last_fullsized":"2023-03-25T04:33:38.333467+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117119,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:36:50 2023","interfaces":[{"interface":"back","average":{"1min":0.254,"5min":0.25600000000000001,"15min":0.27900000000000003},"min":{"1min":0.20499999999999999,"5min":0.16,"15min":0.16},"max":{"1min":0.371,"5min":0.42599999999999999,"15min":0.45100000000000001},"last":0.23000000000000001},{"interface":"front","average":{"1min":0.26800000000000002,"5min":0.26500000000000001,"15min":0.28399999999999997},"min":{"1min":0.19700000000000001,"5min":0.17999999999999999,"15min":0.17499999999999999},"max":{"1min":0.34699999999999998,"5min":0.45200000000000001,"15min":0.46300000000000002},"last":0.20899999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051710,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:36:44 2023","interfaces":[{"interface":"back","average":{"1min":0.28399999999999997,"5min":0.27700000000000002,"15min":0.26100000000000001},"min":{"1min":0.192,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.40899999999999997,"5min":0.42699999999999999,"15min":0.42699999999999999},"last":0.39700000000000002},{"interface":"front","average":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.27500000000000002},"min":{"1min":0.182,"5min":0.182,"15min":0.16600000000000001},"max":{"1min":0.439,"5min":0.45800000000000002,"15min":0.45800000000000002},"last":0.439}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:37:42.400+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)

Ignoring malparsed backtrace: 2023-03-25T04:38:04.766+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:38:04.922 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:38:05.246 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:38:19.605 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":411,"stamp":"2023-03-25T04:38:18.220966+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002368"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":38,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:36:03.347070+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:36:03.347070+0000","last_peered":"2023-03-25T04:36:03.347070+0000","last_clean":"2023-03-25T04:36:03.347070+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:36:03.347070+0000","last_undegraded":"2023-03-25T04:36:03.347070+0000","last_fullsized":"2023-03-25T04:36:03.347070+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:36:03.347025+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:36:03.347025+0000","last_peered":"2023-03-25T04:36:03.347025+0000","last_clean":"2023-03-25T04:36:03.347025+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:36:03.347025+0000","last_undegraded":"2023-03-25T04:36:03.347025+0000","last_fullsized":"2023-03-25T04:36:03.347025+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:23.343585+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:35:23.343585+0000","last_peered":"2023-03-25T04:35:23.343585+0000","last_clean":"2023-03-25T04:35:23.343585+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:35:23.343585+0000","last_undegraded":"2023-03-25T04:35:23.343585+0000","last_fullsized":"2023-03-25T04:35:23.343585+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:35.587025+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:35:35.587025+0000","last_peered":"2023-03-25T04:35:35.587025+0000","last_clean":"2023-03-25T04:35:35.587025+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:35:35.587025+0000","last_undegraded":"2023-03-25T04:35:35.587025+0000","last_fullsized":"2023-03-25T04:35:35.587025+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":38,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:43.345308+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:35:43.345308+0000","last_peered":"2023-03-25T04:35:43.345308+0000","last_clean":"2023-03-25T04:35:43.345308+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:35:43.345308+0000","last_undegraded":"2023-03-25T04:35:43.345308+0000","last_fullsized":"2023-03-25T04:35:43.345308+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:43.345324+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:35:43.345324+0000","last_peered":"2023-03-25T04:35:43.345324+0000","last_clean":"2023-03-25T04:35:43.345324+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:35:43.345324+0000","last_undegraded":"2023-03-25T04:35:43.345324+0000","last_fullsized":"2023-03-25T04:35:43.345324+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117127,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:37:53 2023","interfaces":[{"interface":"back","average":{"1min":0.25800000000000001,"5min":0.253,"15min":0.27700000000000002},"min":{"1min":0.184,"5min":0.16,"15min":0.16},"max":{"1min":0.44600000000000001,"5min":0.44600000000000001,"15min":0.45100000000000001},"last":0.35199999999999998},{"interface":"front","average":{"1min":0.26900000000000002,"5min":0.26500000000000001,"15min":0.28199999999999997},"min":{"1min":0.192,"5min":0.17999999999999999,"15min":0.17499999999999999},"max":{"1min":0.46400000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"last":0.33200000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051717,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:37:44 2023","interfaces":[{"interface":"back","average":{"1min":0.27300000000000002,"5min":0.28199999999999997,"15min":0.26200000000000001},"min":{"1min":0.17899999999999999,"5min":0.17299999999999999,"15min":0.17000000000000001},"max":{"1min":0.41799999999999998,"5min":0.42099999999999999,"15min":0.42699999999999999},"last":0.28699999999999998},{"interface":"front","average":{"1min":0.29399999999999998,"5min":0.29099999999999998,"15min":0.27700000000000002},"min":{"1min":0.19900000000000001,"5min":0.182,"15min":0.16600000000000001},"max":{"1min":0.441,"5min":0.441,"15min":0.45800000000000002},"last":0.26300000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:38:19.718+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:38:20.315+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)

Ignoring malparsed backtrace: 2023-03-25T04:38:44.283 INFO:teuthology.orchestra.run.smithi070.stdout:
, 2023-03-25T04:38:44.283 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":781,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":16,"modified":"2023-03-25T04:38:40.225321+0000","services":{}},"progress_events":{}}
, 2023-03-25T04:38:44.296 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now

Ignoring malparsed backtrace: 2023-03-25T04:39:02.477+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:39:02.578+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:39:03.524+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:39:20.359 INFO:teuthology.orchestra.run.smithi070.stdout:
, 2023-03-25T04:39:20.359 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":818,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":16,"modified":"2023-03-25T04:38:40.225321+0000","services":{}},"progress_events":{}}
, 2023-03-25T04:39:20.372 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now

Ignoring malparsed backtrace: 2023-03-25T04:39:37.527 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:39:37.541 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":450,"stamp":"2023-03-25T04:39:36.235332+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002086"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":38,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:36:03.347070+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:36:03.347070+0000","last_peered":"2023-03-25T04:36:03.347070+0000","last_clean":"2023-03-25T04:36:03.347070+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:36:03.347070+0000","last_undegraded":"2023-03-25T04:36:03.347070+0000","last_fullsized":"2023-03-25T04:36:03.347070+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:36:03.347025+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:36:03.347025+0000","last_peered":"2023-03-25T04:36:03.347025+0000","last_clean":"2023-03-25T04:36:03.347025+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:36:03.347025+0000","last_undegraded":"2023-03-25T04:36:03.347025+0000","last_fullsized":"2023-03-25T04:36:03.347025+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:23.353778+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:37:23.353778+0000","last_peered":"2023-03-25T04:37:23.353778+0000","last_clean":"2023-03-25T04:37:23.353778+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:37:23.353778+0000","last_undegraded":"2023-03-25T04:37:23.353778+0000","last_fullsized":"2023-03-25T04:37:23.353778+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:35.587025+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:35:35.587025+0000","last_peered":"2023-03-25T04:35:35.587025+0000","last_clean":"2023-03-25T04:35:35.587025+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:35:35.587025+0000","last_undegraded":"2023-03-25T04:35:35.587025+0000","last_fullsized":"2023-03-25T04:35:35.587025+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":38,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:43.345308+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:35:43.345308+0000","last_peered":"2023-03-25T04:35:43.345308+0000","last_clean":"2023-03-25T04:35:43.345308+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:35:43.345308+0000","last_undegraded":"2023-03-25T04:35:43.345308+0000","last_fullsized":"2023-03-25T04:35:43.345308+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:43.345324+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:35:43.345324+0000","last_peered":"2023-03-25T04:35:43.345324+0000","last_clean":"2023-03-25T04:35:43.345324+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:35:43.345324+0000","last_undegraded":"2023-03-25T04:35:43.345324+0000","last_fullsized":"2023-03-25T04:35:43.345324+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117143,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:38:54 2023","interfaces":[{"interface":"back","average":{"1min":0.25800000000000001,"5min":0.25700000000000001,"15min":0.27400000000000002},"min":{"1min":0.192,"5min":0.17399999999999999,"15min":0.16},"max":{"1min":0.39400000000000002,"5min":0.44600000000000001,"15min":0.45100000000000001},"last":0.26500000000000001},{"interface":"front","average":{"1min":0.25900000000000001,"5min":0.26700000000000002,"15min":0.27900000000000003},"min":{"1min":0.19500000000000001,"5min":0.188,"15min":0.17499999999999999},"max":{"1min":0.41499999999999998,"5min":0.46400000000000002,"15min":0.46400000000000002},"last":0.29099999999999998}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051733,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:38:45 2023","interfaces":[{"interface":"back","average":{"1min":0.27300000000000002,"5min":0.28100000000000003,"15min":0.26400000000000001},"min":{"1min":0.19400000000000001,"5min":0.17299999999999999,"15min":0.17000000000000001},"max":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"last":0.27300000000000002},{"interface":"front","average":{"1min":0.28799999999999998,"5min":0.29099999999999998,"15min":0.27800000000000002},"min":{"1min":0.19800000000000001,"5min":0.182,"15min":0.16600000000000001},"max":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"last":0.308}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:39:54.416+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:39:54.455 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:39:54.780 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:40:09.288+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:40:09.389+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:40:10.267+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)

Ignoring malparsed backtrace: 2023-03-25T04:40:28.984 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":476,"stamp":"2023-03-25T04:40:28.245911+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002693"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":39,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:38:13.358968+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:38:13.358968+0000","last_peered":"2023-03-25T04:38:13.358968+0000","last_clean":"2023-03-25T04:38:13.358968+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:38:13.358968+0000","last_undegraded":"2023-03-25T04:38:13.358968+0000","last_fullsized":"2023-03-25T04:38:13.358968+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:38:13.358922+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:38:13.358922+0000","last_peered":"2023-03-25T04:38:13.358922+0000","last_clean":"2023-03-25T04:38:13.358922+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:38:13.358922+0000","last_undegraded":"2023-03-25T04:38:13.358922+0000","last_fullsized":"2023-03-25T04:38:13.358922+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:23.353778+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:37:23.353778+0000","last_peered":"2023-03-25T04:37:23.353778+0000","last_clean":"2023-03-25T04:37:23.353778+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:37:23.353778+0000","last_undegraded":"2023-03-25T04:37:23.353778+0000","last_fullsized":"2023-03-25T04:37:23.353778+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:40.598579+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:37:40.598579+0000","last_peered":"2023-03-25T04:37:40.598579+0000","last_clean":"2023-03-25T04:37:40.598579+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:37:40.598579+0000","last_undegraded":"2023-03-25T04:37:40.598579+0000","last_fullsized":"2023-03-25T04:37:40.598579+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":39,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:48.356344+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:37:48.356344+0000","last_peered":"2023-03-25T04:37:48.356344+0000","last_clean":"2023-03-25T04:37:48.356344+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:37:48.356344+0000","last_undegraded":"2023-03-25T04:37:48.356344+0000","last_fullsized":"2023-03-25T04:37:48.356344+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:48.356365+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:37:48.356365+0000","last_peered":"2023-03-25T04:37:48.356365+0000","last_clean":"2023-03-25T04:37:48.356365+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:37:48.356365+0000","last_undegraded":"2023-03-25T04:37:48.356365+0000","last_fullsized":"2023-03-25T04:37:48.356365+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117153,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:39:54 2023","interfaces":[{"interface":"back","average":{"1min":0.25900000000000001,"5min":0.25600000000000001,"15min":0.27100000000000002},"min":{"1min":0.188,"5min":0.17899999999999999,"15min":0.16},"max":{"1min":0.41999999999999998,"5min":0.44600000000000001,"15min":0.45100000000000001},"last":0.255},{"interface":"front","average":{"1min":0.27500000000000002,"5min":0.26600000000000001,"15min":0.27700000000000002},"min":{"1min":0.17699999999999999,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.45600000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"last":0.26800000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051743,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:39:47 2023","interfaces":[{"interface":"back","average":{"1min":0.26300000000000001,"5min":0.27700000000000002,"15min":0.26500000000000001},"min":{"1min":0.187,"5min":0.17299999999999999,"15min":0.17000000000000001},"max":{"1min":0.433,"5min":0.433,"15min":0.433},"last":0.36599999999999999},{"interface":"front","average":{"1min":0.27100000000000002,"5min":0.28799999999999998,"15min":0.27900000000000003},"min":{"1min":0.20100000000000001,"5min":0.182,"15min":0.16600000000000001},"max":{"1min":0.48599999999999999,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.34999999999999998}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:40:28.986 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean
, 2023-03-25T04:40:28.986 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}

Ignoring malparsed backtrace: 2023-03-25T04:40:47.229+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:40:47.503+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:40:48.251+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)

Ignoring malparsed backtrace: 2023-03-25T04:41:03.267 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:41:03.578 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all

Ignoring malparsed backtrace: 2023-03-25T04:41:21.347 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":502,"stamp":"2023-03-25T04:41:20.256204+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002342"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":39,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:38:13.358968+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:38:13.358968+0000","last_peered":"2023-03-25T04:38:13.358968+0000","last_clean":"2023-03-25T04:38:13.358968+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:38:13.358968+0000","last_undegraded":"2023-03-25T04:38:13.358968+0000","last_fullsized":"2023-03-25T04:38:13.358968+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:38:13.358922+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:38:13.358922+0000","last_peered":"2023-03-25T04:38:13.358922+0000","last_clean":"2023-03-25T04:38:13.358922+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:38:13.358922+0000","last_undegraded":"2023-03-25T04:38:13.358922+0000","last_fullsized":"2023-03-25T04:38:13.358922+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:23.353778+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:37:23.353778+0000","last_peered":"2023-03-25T04:37:23.353778+0000","last_clean":"2023-03-25T04:37:23.353778+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:37:23.353778+0000","last_undegraded":"2023-03-25T04:37:23.353778+0000","last_fullsized":"2023-03-25T04:37:23.353778+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:40.598579+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:37:40.598579+0000","last_peered":"2023-03-25T04:37:40.598579+0000","last_clean":"2023-03-25T04:37:40.598579+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:37:40.598579+0000","last_undegraded":"2023-03-25T04:37:40.598579+0000","last_fullsized":"2023-03-25T04:37:40.598579+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":39,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:48.356344+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:37:48.356344+0000","last_peered":"2023-03-25T04:37:48.356344+0000","last_clean":"2023-03-25T04:37:48.356344+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:37:48.356344+0000","last_undegraded":"2023-03-25T04:37:48.356344+0000","last_fullsized":"2023-03-25T04:37:48.356344+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:48.356365+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:37:48.356365+0000","last_peered":"2023-03-25T04:37:48.356365+0000","last_clean":"2023-03-25T04:37:48.356365+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:37:48.356365+0000","last_undegraded":"2023-03-25T04:37:48.356365+0000","last_fullsized":"2023-03-25T04:37:48.356365+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117163,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:40:56 2023","interfaces":[{"interface":"back","average":{"1min":0.28799999999999998,"5min":0.26300000000000001,"15min":0.27100000000000002},"min":{"1min":0.188,"5min":0.184,"15min":0.16},"max":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.17499999999999999},{"interface":"front","average":{"1min":0.29599999999999999,"5min":0.27300000000000002,"15min":0.27700000000000002},"min":{"1min":0.20599999999999999,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.43099999999999999,"5min":0.46400000000000002,"15min":0.46400000000000002},"last":0.20699999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051754,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:40:52 2023","interfaces":[{"interface":"back","average":{"1min":0.28599999999999998,"5min":0.27500000000000002,"15min":0.26700000000000002},"min":{"1min":0.17899999999999999,"5min":0.17899999999999999,"15min":0.17000000000000001},"max":{"1min":0.44,"5min":0.44,"15min":0.44},"last":0.23699999999999999},{"interface":"front","average":{"1min":0.29799999999999999,"5min":0.28699999999999998,"15min":0.28100000000000003},"min":{"1min":0.20300000000000001,"5min":0.182,"15min":0.16600000000000001},"max":{"1min":0.47599999999999998,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.28000000000000003}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:41:21.349 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean
, 2023-03-25T04:41:21.349 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}

Ignoring malparsed backtrace: 2023-03-25T04:41:39.867+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:41:40.383+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:41:40.905+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:41:58.219 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json
, 2023-03-25T04:41:58.319+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:41:58.644 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:42:18.193 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":530,"stamp":"2023-03-25T04:42:16.266659+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002356"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":39,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:38:13.358968+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:38:13.358968+0000","last_peered":"2023-03-25T04:38:13.358968+0000","last_clean":"2023-03-25T04:38:13.358968+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:38:13.358968+0000","last_undegraded":"2023-03-25T04:38:13.358968+0000","last_fullsized":"2023-03-25T04:38:13.358968+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:38:13.358922+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:38:13.358922+0000","last_peered":"2023-03-25T04:38:13.358922+0000","last_clean":"2023-03-25T04:38:13.358922+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:38:13.358922+0000","last_undegraded":"2023-03-25T04:38:13.358922+0000","last_fullsized":"2023-03-25T04:38:13.358922+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:23.364593+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:39:23.364593+0000","last_peered":"2023-03-25T04:39:23.364593+0000","last_clean":"2023-03-25T04:39:23.364593+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:39:23.364593+0000","last_undegraded":"2023-03-25T04:39:23.364593+0000","last_fullsized":"2023-03-25T04:39:23.364593+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:45.608475+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:39:45.608475+0000","last_peered":"2023-03-25T04:39:45.608475+0000","last_clean":"2023-03-25T04:39:45.608475+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:39:45.608475+0000","last_undegraded":"2023-03-25T04:39:45.608475+0000","last_fullsized":"2023-03-25T04:39:45.608475+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":40,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:53.367398+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:39:53.367398+0000","last_peered":"2023-03-25T04:39:53.367398+0000","last_clean":"2023-03-25T04:39:53.367398+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:39:53.367398+0000","last_undegraded":"2023-03-25T04:39:53.367398+0000","last_fullsized":"2023-03-25T04:39:53.367398+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:53.367424+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:39:53.367424+0000","last_peered":"2023-03-25T04:39:53.367424+0000","last_clean":"2023-03-25T04:39:53.367424+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:39:53.367424+0000","last_undegraded":"2023-03-25T04:39:53.367424+0000","last_fullsized":"2023-03-25T04:39:53.367424+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117175,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:41:58 2023","interfaces":[{"interface":"back","average":{"1min":0.25800000000000001,"5min":0.26400000000000001,"15min":0.26800000000000002},"min":{"1min":0.17499999999999999,"5min":0.17499999999999999,"15min":0.16},"max":{"1min":0.39100000000000001,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.22900000000000001},{"interface":"front","average":{"1min":0.26300000000000001,"5min":0.27200000000000002,"15min":0.27400000000000002},"min":{"1min":0.20599999999999999,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.35699999999999998,"5min":0.46400000000000002,"15min":0.46400000000000002},"last":0.26400000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051765,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:41:52 2023","interfaces":[{"interface":"back","average":{"1min":0.26500000000000001,"5min":0.27200000000000002,"15min":0.26800000000000002},"min":{"1min":0.17000000000000001,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.39200000000000002,"5min":0.44,"15min":0.44},"last":0.23899999999999999},{"interface":"front","average":{"1min":0.28100000000000003,"5min":0.28599999999999998,"15min":0.28199999999999997},"min":{"1min":0.19400000000000001,"5min":0.19400000000000001,"15min":0.16600000000000001},"max":{"1min":0.41399999999999998,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.254}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:42:18.297+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:42:18.958+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:42:35.512 INFO:teuthology.orchestra.run.smithi070.stdout:
, 2023-03-25T04:42:35.512 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":1013,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":17,"modified":"2023-03-25T04:41:54.262744+0000","services":{}},"progress_events":{}}
, 2023-03-25T04:42:35.525 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:42:52.741 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":548,"stamp":"2023-03-25T04:42:52.273601+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002221"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":40,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:40:23.370056+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:40:23.370056+0000","last_peered":"2023-03-25T04:40:23.370056+0000","last_clean":"2023-03-25T04:40:23.370056+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:40:23.370056+0000","last_undegraded":"2023-03-25T04:40:23.370056+0000","last_fullsized":"2023-03-25T04:40:23.370056+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:40:23.370039+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:40:23.370039+0000","last_peered":"2023-03-25T04:40:23.370039+0000","last_clean":"2023-03-25T04:40:23.370039+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:40:23.370039+0000","last_undegraded":"2023-03-25T04:40:23.370039+0000","last_fullsized":"2023-03-25T04:40:23.370039+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:23.364593+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:39:23.364593+0000","last_peered":"2023-03-25T04:39:23.364593+0000","last_clean":"2023-03-25T04:39:23.364593+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:39:23.364593+0000","last_undegraded":"2023-03-25T04:39:23.364593+0000","last_fullsized":"2023-03-25T04:39:23.364593+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:45.608475+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:39:45.608475+0000","last_peered":"2023-03-25T04:39:45.608475+0000","last_clean":"2023-03-25T04:39:45.608475+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:39:45.608475+0000","last_undegraded":"2023-03-25T04:39:45.608475+0000","last_fullsized":"2023-03-25T04:39:45.608475+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":40,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:53.367398+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:39:53.367398+0000","last_peered":"2023-03-25T04:39:53.367398+0000","last_clean":"2023-03-25T04:39:53.367398+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:39:53.367398+0000","last_undegraded":"2023-03-25T04:39:53.367398+0000","last_fullsized":"2023-03-25T04:39:53.367398+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:53.367424+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:39:53.367424+0000","last_peered":"2023-03-25T04:39:53.367424+0000","last_clean":"2023-03-25T04:39:53.367424+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:39:53.367424+0000","last_undegraded":"2023-03-25T04:39:53.367424+0000","last_fullsized":"2023-03-25T04:39:53.367424+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117182,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:41:58 2023","interfaces":[{"interface":"back","average":{"1min":0.25800000000000001,"5min":0.26400000000000001,"15min":0.26800000000000002},"min":{"1min":0.17499999999999999,"5min":0.17499999999999999,"15min":0.16},"max":{"1min":0.39100000000000001,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.19600000000000001},{"interface":"front","average":{"1min":0.26300000000000001,"5min":0.27200000000000002,"15min":0.27400000000000002},"min":{"1min":0.20599999999999999,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.35699999999999998,"5min":0.46400000000000002,"15min":0.46400000000000002},"last":0.219}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051773,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:41:52 2023","interfaces":[{"interface":"back","average":{"1min":0.26500000000000001,"5min":0.27200000000000002,"15min":0.26800000000000002},"min":{"1min":0.17000000000000001,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.39200000000000002,"5min":0.44,"15min":0.44},"last":0.19},{"interface":"front","average":{"1min":0.28100000000000003,"5min":0.28599999999999998,"15min":0.28199999999999997},"min":{"1min":0.19400000000000001,"5min":0.19400000000000001,"15min":0.16600000000000001},"max":{"1min":0.41399999999999998,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.17599999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:42:52.742 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean

Ignoring malparsed backtrace: 2023-03-25T04:43:09.973 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:43:09.986 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":556,"stamp":"2023-03-25T04:43:08.276556+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002359"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":40,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:40:23.370056+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:40:23.370056+0000","last_peered":"2023-03-25T04:40:23.370056+0000","last_clean":"2023-03-25T04:40:23.370056+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:40:23.370056+0000","last_undegraded":"2023-03-25T04:40:23.370056+0000","last_fullsized":"2023-03-25T04:40:23.370056+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:40:23.370039+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:40:23.370039+0000","last_peered":"2023-03-25T04:40:23.370039+0000","last_clean":"2023-03-25T04:40:23.370039+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:40:23.370039+0000","last_undegraded":"2023-03-25T04:40:23.370039+0000","last_fullsized":"2023-03-25T04:40:23.370039+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:23.364593+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:39:23.364593+0000","last_peered":"2023-03-25T04:39:23.364593+0000","last_clean":"2023-03-25T04:39:23.364593+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:39:23.364593+0000","last_undegraded":"2023-03-25T04:39:23.364593+0000","last_fullsized":"2023-03-25T04:39:23.364593+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:45.608475+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:39:45.608475+0000","last_peered":"2023-03-25T04:39:45.608475+0000","last_clean":"2023-03-25T04:39:45.608475+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:39:45.608475+0000","last_undegraded":"2023-03-25T04:39:45.608475+0000","last_fullsized":"2023-03-25T04:39:45.608475+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":40,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:53.367398+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:39:53.367398+0000","last_peered":"2023-03-25T04:39:53.367398+0000","last_clean":"2023-03-25T04:39:53.367398+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:39:53.367398+0000","last_undegraded":"2023-03-25T04:39:53.367398+0000","last_fullsized":"2023-03-25T04:39:53.367398+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:53.367424+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:39:53.367424+0000","last_peered":"2023-03-25T04:39:53.367424+0000","last_clean":"2023-03-25T04:39:53.367424+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:39:53.367424+0000","last_undegraded":"2023-03-25T04:39:53.367424+0000","last_fullsized":"2023-03-25T04:39:53.367424+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117185,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:43:00 2023","interfaces":[{"interface":"back","average":{"1min":0.27400000000000002,"5min":0.26700000000000002,"15min":0.26600000000000001},"min":{"1min":0.19600000000000001,"5min":0.17499999999999999,"15min":0.16},"max":{"1min":0.42999999999999999,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.27800000000000002},{"interface":"front","average":{"1min":0.28199999999999997,"5min":0.27500000000000002,"15min":0.27300000000000002},"min":{"1min":0.185,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.47199999999999998,"5min":0.47199999999999998,"15min":0.47199999999999998},"last":0.247}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051776,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:42:54 2023","interfaces":[{"interface":"back","average":{"1min":0.26300000000000001,"5min":0.27000000000000002,"15min":0.26900000000000002},"min":{"1min":0.17399999999999999,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.438,"5min":0.44,"15min":0.44},"last":0.20100000000000001},{"interface":"front","average":{"1min":0.27400000000000002,"5min":0.28199999999999997,"15min":0.28299999999999997},"min":{"1min":0.17599999999999999,"5min":0.17599999999999999,"15min":0.16600000000000001},"max":{"1min":0.41599999999999998,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.23499999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:43:25.042+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:43:25.411+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:43:26.013+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:43:47.937+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:43:48.534+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:43:48.889+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:44:09.318 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:44:09.331 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":586,"stamp":"2023-03-25T04:44:08.288933+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002697"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":40,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:40:23.370056+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:40:23.370056+0000","last_peered":"2023-03-25T04:40:23.370056+0000","last_clean":"2023-03-25T04:40:23.370056+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:40:23.370056+0000","last_undegraded":"2023-03-25T04:40:23.370056+0000","last_fullsized":"2023-03-25T04:40:23.370056+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:40:23.370039+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:40:23.370039+0000","last_peered":"2023-03-25T04:40:23.370039+0000","last_clean":"2023-03-25T04:40:23.370039+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:40:23.370039+0000","last_undegraded":"2023-03-25T04:40:23.370039+0000","last_fullsized":"2023-03-25T04:40:23.370039+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:23.375106+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:41:23.375106+0000","last_peered":"2023-03-25T04:41:23.375106+0000","last_clean":"2023-03-25T04:41:23.375106+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:41:23.375106+0000","last_undegraded":"2023-03-25T04:41:23.375106+0000","last_fullsized":"2023-03-25T04:41:23.375106+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:50.618930+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:41:50.618930+0000","last_peered":"2023-03-25T04:41:50.618930+0000","last_clean":"2023-03-25T04:41:50.618930+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:41:50.618930+0000","last_undegraded":"2023-03-25T04:41:50.618930+0000","last_fullsized":"2023-03-25T04:41:50.618930+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":41,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:58.377821+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:41:58.377821+0000","last_peered":"2023-03-25T04:41:58.377821+0000","last_clean":"2023-03-25T04:41:58.377821+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:41:58.377821+0000","last_undegraded":"2023-03-25T04:41:58.377821+0000","last_fullsized":"2023-03-25T04:41:58.377821+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:58.377836+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:41:58.377836+0000","last_peered":"2023-03-25T04:41:58.377836+0000","last_clean":"2023-03-25T04:41:58.377836+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:41:58.377836+0000","last_undegraded":"2023-03-25T04:41:58.377836+0000","last_fullsized":"2023-03-25T04:41:58.377836+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117197,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:44:01 2023","interfaces":[{"interface":"back","average":{"1min":0.27200000000000002,"5min":0.27000000000000002,"15min":0.26400000000000001},"min":{"1min":0.20399999999999999,"5min":0.17499999999999999,"15min":0.16},"max":{"1min":0.38400000000000001,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.245},{"interface":"front","average":{"1min":0.27600000000000002,"5min":0.27800000000000002,"15min":0.27100000000000002},"min":{"1min":0.23100000000000001,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.40400000000000003,"5min":0.47199999999999998,"15min":0.47199999999999998},"last":0.27500000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051788,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:43:55 2023","interfaces":[{"interface":"back","average":{"1min":0.26100000000000001,"5min":0.26700000000000002,"15min":0.27000000000000002},"min":{"1min":0.17799999999999999,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.374,"5min":0.44,"15min":0.44},"last":0.30299999999999999},{"interface":"front","average":{"1min":0.28299999999999997,"5min":0.28100000000000003,"15min":0.28399999999999997},"min":{"1min":0.20899999999999999,"5min":0.17599999999999999,"15min":0.16600000000000001},"max":{"1min":0.40100000000000002,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.27800000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:44:28.923 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":1126,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":19,"modified":"2023-03-25T04:43:20.278959+0000","services":{}},"progress_events":{}}
, 2023-03-25T04:44:28.936 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now
, 2023-03-25T04:44:28.937 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace: 2023-03-25T04:44:48.786 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:44:48.800 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":606,"stamp":"2023-03-25T04:44:48.296805+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002182"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":41,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:42:34.382196+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:42:34.382196+0000","last_peered":"2023-03-25T04:42:34.382196+0000","last_clean":"2023-03-25T04:42:34.382196+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:42:34.382196+0000","last_undegraded":"2023-03-25T04:42:34.382196+0000","last_fullsized":"2023-03-25T04:42:34.382196+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:42:29.381744+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:42:29.381744+0000","last_peered":"2023-03-25T04:42:29.381744+0000","last_clean":"2023-03-25T04:42:29.381744+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:42:29.381744+0000","last_undegraded":"2023-03-25T04:42:29.381744+0000","last_fullsized":"2023-03-25T04:42:29.381744+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:23.375106+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:41:23.375106+0000","last_peered":"2023-03-25T04:41:23.375106+0000","last_clean":"2023-03-25T04:41:23.375106+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:41:23.375106+0000","last_undegraded":"2023-03-25T04:41:23.375106+0000","last_fullsized":"2023-03-25T04:41:23.375106+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:50.618930+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:41:50.618930+0000","last_peered":"2023-03-25T04:41:50.618930+0000","last_clean":"2023-03-25T04:41:50.618930+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:41:50.618930+0000","last_undegraded":"2023-03-25T04:41:50.618930+0000","last_fullsized":"2023-03-25T04:41:50.618930+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":41,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:58.377821+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:41:58.377821+0000","last_peered":"2023-03-25T04:41:58.377821+0000","last_clean":"2023-03-25T04:41:58.377821+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:41:58.377821+0000","last_undegraded":"2023-03-25T04:41:58.377821+0000","last_fullsized":"2023-03-25T04:41:58.377821+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:58.377836+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:41:58.377836+0000","last_peered":"2023-03-25T04:41:58.377836+0000","last_clean":"2023-03-25T04:41:58.377836+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:41:58.377836+0000","last_undegraded":"2023-03-25T04:41:58.377836+0000","last_fullsized":"2023-03-25T04:41:58.377836+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117206,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:44:01 2023","interfaces":[{"interface":"back","average":{"1min":0.27200000000000002,"5min":0.27000000000000002,"15min":0.26400000000000001},"min":{"1min":0.20399999999999999,"5min":0.17499999999999999,"15min":0.16},"max":{"1min":0.38400000000000001,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.216},{"interface":"front","average":{"1min":0.27600000000000002,"5min":0.27800000000000002,"15min":0.27100000000000002},"min":{"1min":0.23100000000000001,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.40400000000000003,"5min":0.47199999999999998,"15min":0.47199999999999998},"last":0.23999999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051796,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:43:55 2023","interfaces":[{"interface":"back","average":{"1min":0.26100000000000001,"5min":0.26700000000000002,"15min":0.27000000000000002},"min":{"1min":0.17799999999999999,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.374,"5min":0.44,"15min":0.44},"last":0.25600000000000001},{"interface":"front","average":{"1min":0.28299999999999997,"5min":0.28100000000000003,"15min":0.28399999999999997},"min":{"1min":0.20899999999999999,"5min":0.17599999999999999,"15min":0.16600000000000001},"max":{"1min":0.40100000000000002,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.28000000000000003}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:45:05.762 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:45:06.073 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all

Ignoring malparsed backtrace: 2023-03-25T04:45:25.658 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":624,"stamp":"2023-03-25T04:45:24.304294+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002366"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":41,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:42:34.382196+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:42:34.382196+0000","last_peered":"2023-03-25T04:42:34.382196+0000","last_clean":"2023-03-25T04:42:34.382196+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:42:34.382196+0000","last_undegraded":"2023-03-25T04:42:34.382196+0000","last_fullsized":"2023-03-25T04:42:34.382196+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:42:29.381744+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:42:29.381744+0000","last_peered":"2023-03-25T04:42:29.381744+0000","last_clean":"2023-03-25T04:42:29.381744+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:42:29.381744+0000","last_undegraded":"2023-03-25T04:42:29.381744+0000","last_fullsized":"2023-03-25T04:42:29.381744+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:23.375106+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:41:23.375106+0000","last_peered":"2023-03-25T04:41:23.375106+0000","last_clean":"2023-03-25T04:41:23.375106+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:41:23.375106+0000","last_undegraded":"2023-03-25T04:41:23.375106+0000","last_fullsized":"2023-03-25T04:41:23.375106+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:50.618930+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:41:50.618930+0000","last_peered":"2023-03-25T04:41:50.618930+0000","last_clean":"2023-03-25T04:41:50.618930+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:41:50.618930+0000","last_undegraded":"2023-03-25T04:41:50.618930+0000","last_fullsized":"2023-03-25T04:41:50.618930+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":41,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:58.377821+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:41:58.377821+0000","last_peered":"2023-03-25T04:41:58.377821+0000","last_clean":"2023-03-25T04:41:58.377821+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:41:58.377821+0000","last_undegraded":"2023-03-25T04:41:58.377821+0000","last_fullsized":"2023-03-25T04:41:58.377821+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:58.377836+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:41:58.377836+0000","last_peered":"2023-03-25T04:41:58.377836+0000","last_clean":"2023-03-25T04:41:58.377836+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:41:58.377836+0000","last_undegraded":"2023-03-25T04:41:58.377836+0000","last_fullsized":"2023-03-25T04:41:58.377836+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117213,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:45:01 2023","interfaces":[{"interface":"back","average":{"1min":0.251,"5min":0.26800000000000002,"15min":0.26400000000000001},"min":{"1min":0.183,"5min":0.17499999999999999,"15min":0.16},"max":{"1min":0.41499999999999998,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.26900000000000002},{"interface":"front","average":{"1min":0.28000000000000003,"5min":0.27900000000000003,"15min":0.27100000000000002},"min":{"1min":0.20999999999999999,"5min":0.185,"15min":0.17499999999999999},"max":{"1min":0.44400000000000001,"5min":0.47199999999999998,"15min":0.47199999999999998},"last":0.27900000000000003}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051803,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:44:55 2023","interfaces":[{"interface":"back","average":{"1min":0.252,"5min":0.26500000000000001,"15min":0.26900000000000002},"min":{"1min":0.183,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.39700000000000002,"5min":0.44,"15min":0.44},"last":0.20699999999999999},{"interface":"front","average":{"1min":0.27000000000000002,"5min":0.28100000000000003,"15min":0.28299999999999997},"min":{"1min":0.20300000000000001,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":0.42099999999999999,"5min":0.47599999999999998,"15min":0.48599999999999999},"last":0.23599999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:45:25.659 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:45:25.969 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:45:42.735+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:45:42.744+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:45:42.897 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace: 2023-03-25T04:46:00.183 INFO:teuthology.orchestra.run.smithi070.stdout:
, 2023-03-25T04:46:00.183 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":1217,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":20,"modified":"2023-03-25T04:45:42.307700+0000","services":{"mgr":{"daemons":{"summary":"","x":{"start_epoch":0,"start_stamp":"0.000000","gid":0,"addr":"(unrecognized address family 0)/0","metadata":{},"task_status":{}}}}}},"progress_events":{}}
, 2023-03-25T04:46:00.197 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now

Ignoring malparsed backtrace: 2023-03-25T04:46:19.910+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:46:20.039 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all

Ignoring malparsed backtrace: 2023-03-25T04:46:39.611 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}
, 2023-03-25T04:46:39.611 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json
, 2023-03-25T04:46:39.613+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:46:58.485+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:46:58.764+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:46:59.519+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:47:14.132 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:47:14.146 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":678,"stamp":"2023-03-25T04:47:12.325044+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002571"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":42,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:44.393426+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:44:44.393426+0000","last_peered":"2023-03-25T04:44:44.393426+0000","last_clean":"2023-03-25T04:44:44.393426+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:44:44.393426+0000","last_undegraded":"2023-03-25T04:44:44.393426+0000","last_fullsized":"2023-03-25T04:44:44.393426+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:39.393044+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:44:39.393044+0000","last_peered":"2023-03-25T04:44:39.393044+0000","last_clean":"2023-03-25T04:44:39.393044+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:44:39.393044+0000","last_undegraded":"2023-03-25T04:44:39.393044+0000","last_fullsized":"2023-03-25T04:44:39.393044+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:43:24.386177+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:43:24.386177+0000","last_peered":"2023-03-25T04:43:24.386177+0000","last_clean":"2023-03-25T04:43:24.386177+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:43:24.386177+0000","last_undegraded":"2023-03-25T04:43:24.386177+0000","last_fullsized":"2023-03-25T04:43:24.386177+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:43:55.629798+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:43:55.629798+0000","last_peered":"2023-03-25T04:43:55.629798+0000","last_clean":"2023-03-25T04:43:55.629798+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:43:55.629798+0000","last_undegraded":"2023-03-25T04:43:55.629798+0000","last_fullsized":"2023-03-25T04:43:55.629798+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":42,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:04.389904+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:44:04.389904+0000","last_peered":"2023-03-25T04:44:04.389904+0000","last_clean":"2023-03-25T04:44:04.389904+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:44:04.389904+0000","last_undegraded":"2023-03-25T04:44:04.389904+0000","last_fullsized":"2023-03-25T04:44:04.389904+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:04.389923+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:44:04.389923+0000","last_peered":"2023-03-25T04:44:04.389923+0000","last_clean":"2023-03-25T04:44:04.389923+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:44:04.389923+0000","last_undegraded":"2023-03-25T04:44:04.389923+0000","last_fullsized":"2023-03-25T04:44:04.389923+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117235,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:47:05 2023","interfaces":[{"interface":"back","average":{"1min":0.27800000000000002,"5min":0.27000000000000002,"15min":0.26300000000000001},"min":{"1min":0.20100000000000001,"5min":0.16900000000000001,"15min":0.16},"max":{"1min":0.39200000000000002,"5min":0.42999999999999999,"15min":0.45600000000000002},"last":0.379},{"interface":"front","average":{"1min":0.28399999999999997,"5min":0.28100000000000003,"15min":0.27200000000000002},"min":{"1min":0.17799999999999999,"5min":0.17799999999999999,"15min":0.17699999999999999},"max":{"1min":0.42599999999999999,"5min":0.47199999999999998,"15min":0.47199999999999998},"last":0.39800000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051825,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:47:01 2023","interfaces":[{"interface":"back","average":{"1min":0.27300000000000002,"5min":0.26100000000000001,"15min":0.27000000000000002},"min":{"1min":0.186,"5min":0.17399999999999999,"15min":0.17000000000000001},"max":{"1min":0.40400000000000003,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.23899999999999999},{"interface":"front","average":{"1min":0.28699999999999998,"5min":0.27500000000000002,"15min":0.28299999999999997},"min":{"1min":0.186,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.48599999999999999},"last":0.19600000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:47:31.198 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json
, 2023-03-25T04:47:31.411+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:47:31.789 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:47:49.509 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":696,"stamp":"2023-03-25T04:47:48.332093+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002697"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":42,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:44.393426+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:44:44.393426+0000","last_peered":"2023-03-25T04:44:44.393426+0000","last_clean":"2023-03-25T04:44:44.393426+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:44:44.393426+0000","last_undegraded":"2023-03-25T04:44:44.393426+0000","last_fullsized":"2023-03-25T04:44:44.393426+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:39.393044+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:44:39.393044+0000","last_peered":"2023-03-25T04:44:39.393044+0000","last_clean":"2023-03-25T04:44:39.393044+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:44:39.393044+0000","last_undegraded":"2023-03-25T04:44:39.393044+0000","last_fullsized":"2023-03-25T04:44:39.393044+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:45:24.396717+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:45:24.396717+0000","last_peered":"2023-03-25T04:45:24.396717+0000","last_clean":"2023-03-25T04:45:24.396717+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:45:24.396717+0000","last_undegraded":"2023-03-25T04:45:24.396717+0000","last_fullsized":"2023-03-25T04:45:24.396717+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:43:55.629798+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:43:55.629798+0000","last_peered":"2023-03-25T04:43:55.629798+0000","last_clean":"2023-03-25T04:43:55.629798+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:43:55.629798+0000","last_undegraded":"2023-03-25T04:43:55.629798+0000","last_fullsized":"2023-03-25T04:43:55.629798+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":42,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:04.389904+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:44:04.389904+0000","last_peered":"2023-03-25T04:44:04.389904+0000","last_clean":"2023-03-25T04:44:04.389904+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:44:04.389904+0000","last_undegraded":"2023-03-25T04:44:04.389904+0000","last_fullsized":"2023-03-25T04:44:04.389904+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:04.389923+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:44:04.389923+0000","last_peered":"2023-03-25T04:44:04.389923+0000","last_clean":"2023-03-25T04:44:04.389923+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:44:04.389923+0000","last_undegraded":"2023-03-25T04:44:04.389923+0000","last_fullsized":"2023-03-25T04:44:04.389923+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117242,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:47:05 2023","interfaces":[{"interface":"back","average":{"1min":0.27800000000000002,"5min":0.27000000000000002,"15min":0.26300000000000001},"min":{"1min":0.20100000000000001,"5min":0.16900000000000001,"15min":0.16},"max":{"1min":0.39200000000000002,"5min":0.42999999999999999,"15min":0.45600000000000002},"last":0.40500000000000003},{"interface":"front","average":{"1min":0.28399999999999997,"5min":0.28100000000000003,"15min":0.27200000000000002},"min":{"1min":0.17799999999999999,"5min":0.17799999999999999,"15min":0.17699999999999999},"max":{"1min":0.42599999999999999,"5min":0.47199999999999998,"15min":0.47199999999999998},"last":0.42399999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051834,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:47:01 2023","interfaces":[{"interface":"back","average":{"1min":0.27300000000000002,"5min":0.26100000000000001,"15min":0.27000000000000002},"min":{"1min":0.186,"5min":0.17399999999999999,"15min":0.17000000000000001},"max":{"1min":0.40400000000000003,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.25600000000000001},{"interface":"front","average":{"1min":0.28699999999999998,"5min":0.27500000000000002,"15min":0.28299999999999997},"min":{"1min":0.186,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.48599999999999999},"last":0.22800000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:47:49.510 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace:     ctx.managers[config['cluster']].wait_for_clean()
,   File "/home/teuthworker/src/github.com_ceph_ceph-c_511015c23eef0684a64171b710e432448df2b7c7/qa/tasks/ceph_manager.py", line 2747, in wait_for_clean
,     assert time.time() - start < timeout, \

Ignoring malparsed backtrace:  ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
,  1: /lib64/libpthread.so.0(+0x12cf0) [0x7f1453c2fcf0]
,  2: __pthread_mutex_lock()

Ignoring malparsed backtrace: 2023-03-25T04:28:05.172 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":142,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":5,"modified":"2023-03-25T04:27:12.092098+0000","services":{}},"progress_events":{}}
, 2023-03-25T04:28:05.185 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now
, 2023-03-25T04:28:05.186 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace: 2023-03-25T04:28:28.360 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":116,"stamp":"2023-03-25T04:28:28.108007+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002578"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.314661+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:27:45.314661+0000","last_peered":"2023-03-25T04:27:45.314661+0000","last_clean":"2023-03-25T04:27:45.314661+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:27:45.314661+0000","last_undegraded":"2023-03-25T04:27:45.314661+0000","last_fullsized":"2023-03-25T04:27:45.314661+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":25,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.310745+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:27:44.310745+0000","last_peered":"2023-03-25T04:27:44.310745+0000","last_clean":"2023-03-25T04:27:44.310745+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:27:44.310745+0000","last_undegraded":"2023-03-25T04:27:44.310745+0000","last_fullsized":"2023-03-25T04:27:44.310745+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":25,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.310650+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:27:44.310650+0000","last_peered":"2023-03-25T04:27:44.310650+0000","last_clean":"2023-03-25T04:27:44.310650+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:27:44.310650+0000","last_undegraded":"2023-03-25T04:27:44.310650+0000","last_fullsized":"2023-03-25T04:27:44.310650+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":26,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.312962+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:27:44.312962+0000","last_peered":"2023-03-25T04:27:44.312962+0000","last_clean":"2023-03-25T04:27:44.312962+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:27:44.312962+0000","last_undegraded":"2023-03-25T04:27:44.312962+0000","last_fullsized":"2023-03-25T04:27:44.312962+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315064+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:27:45.315064+0000","last_peered":"2023-03-25T04:27:45.315064+0000","last_clean":"2023-03-25T04:27:45.315064+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:27:45.315064+0000","last_undegraded":"2023-03-25T04:27:45.315064+0000","last_fullsized":"2023-03-25T04:27:45.315064+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":25,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.310707+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:27:44.310707+0000","last_peered":"2023-03-25T04:27:44.310707+0000","last_clean":"2023-03-25T04:27:44.310707+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:27:44.310707+0000","last_undegraded":"2023-03-25T04:27:44.310707+0000","last_fullsized":"2023-03-25T04:27:44.310707+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117007,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.375}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051597,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.186}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:28:28.362 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean
, 2023-03-25T04:28:28.362 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:28:45.655 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":124,"stamp":"2023-03-25T04:28:44.111265+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002492"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.314661+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:27:45.314661+0000","last_peered":"2023-03-25T04:27:45.314661+0000","last_clean":"2023-03-25T04:27:45.314661+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:27:45.314661+0000","last_undegraded":"2023-03-25T04:27:45.314661+0000","last_fullsized":"2023-03-25T04:27:45.314661+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":25,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.310745+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:27:44.310745+0000","last_peered":"2023-03-25T04:27:44.310745+0000","last_clean":"2023-03-25T04:27:44.310745+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:27:44.310745+0000","last_undegraded":"2023-03-25T04:27:44.310745+0000","last_fullsized":"2023-03-25T04:27:44.310745+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":25,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.310650+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:27:44.310650+0000","last_peered":"2023-03-25T04:27:44.310650+0000","last_clean":"2023-03-25T04:27:44.310650+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:27:44.310650+0000","last_undegraded":"2023-03-25T04:27:44.310650+0000","last_fullsized":"2023-03-25T04:27:44.310650+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":26,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.312962+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:27:44.312962+0000","last_peered":"2023-03-25T04:27:44.312962+0000","last_clean":"2023-03-25T04:27:44.312962+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:27:44.312962+0000","last_undegraded":"2023-03-25T04:27:44.312962+0000","last_fullsized":"2023-03-25T04:27:44.312962+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315064+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:27:45.315064+0000","last_peered":"2023-03-25T04:27:45.315064+0000","last_clean":"2023-03-25T04:27:45.315064+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:27:45.315064+0000","last_undegraded":"2023-03-25T04:27:45.315064+0000","last_fullsized":"2023-03-25T04:27:45.315064+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":25,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.310707+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:27:44.310707+0000","last_peered":"2023-03-25T04:27:44.310707+0000","last_clean":"2023-03-25T04:27:44.310707+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:27:44.310707+0000","last_undegraded":"2023-03-25T04:27:44.310707+0000","last_fullsized":"2023-03-25T04:27:44.310707+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117011,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:28:31 2023","interfaces":[{"interface":"back","average":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.36899999999999999},{"interface":"front","average":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"min":{"1min":0.19,"5min":0.19,"15min":0.19},"max":{"1min":0.46300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.39800000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051601,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:28:24 2023","interfaces":[{"interface":"back","average":{"1min":0.249,"5min":0.249,"15min":0.249},"min":{"1min":0.17699999999999999,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"last":0.254},{"interface":"front","average":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"min":{"1min":0.20200000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"last":0.312}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:28:45.656 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace: 2023-03-25T04:29:05.171 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}
, 2023-03-25T04:29:05.171 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json
, 2023-03-25T04:29:05.561 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:29:22.465 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:29:22.773 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all

Ignoring malparsed backtrace: 2023-03-25T04:29:42.405 INFO:teuthology.orchestra.run.smithi070.stdout:
, 2023-03-25T04:29:42.406 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":240,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":9,"modified":"2023-03-25T04:29:32.120358+0000","services":{"mon":{"daemons":{"summary":"","a":{"start_epoch":0,"start_stamp":"0.000000","gid":0,"addr":"(unrecognized address family 0)/0","metadata":{},"task_status":{}}}}}},"progress_events":{}}
, 2023-03-25T04:29:42.419 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:29:59.627 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":161,"stamp":"2023-03-25T04:29:58.125186+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002398"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315468+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:27:45.315468+0000","last_peered":"2023-03-25T04:27:45.315468+0000","last_clean":"2023-03-25T04:27:45.315468+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:27:45.315468+0000","last_undegraded":"2023-03-25T04:27:45.315468+0000","last_fullsized":"2023-03-25T04:27:45.315468+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312845+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:27:45.312845+0000","last_peered":"2023-03-25T04:27:45.312845+0000","last_clean":"2023-03-25T04:27:45.312845+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:27:45.312845+0000","last_undegraded":"2023-03-25T04:27:45.312845+0000","last_fullsized":"2023-03-25T04:27:45.312845+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312882+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:27:45.312882+0000","last_peered":"2023-03-25T04:27:45.312882+0000","last_clean":"2023-03-25T04:27:45.312882+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:27:45.312882+0000","last_undegraded":"2023-03-25T04:27:45.312882+0000","last_fullsized":"2023-03-25T04:27:45.312882+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.311730+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:27:45.311730+0000","last_peered":"2023-03-25T04:27:45.311730+0000","last_clean":"2023-03-25T04:27:45.311730+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:27:45.311730+0000","last_undegraded":"2023-03-25T04:27:45.311730+0000","last_fullsized":"2023-03-25T04:27:45.311730+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315735+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:27:45.315735+0000","last_peered":"2023-03-25T04:27:45.315735+0000","last_clean":"2023-03-25T04:27:45.315735+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:27:45.315735+0000","last_undegraded":"2023-03-25T04:27:45.315735+0000","last_fullsized":"2023-03-25T04:27:45.315735+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312833+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:27:45.312833+0000","last_peered":"2023-03-25T04:27:45.312833+0000","last_clean":"2023-03-25T04:27:45.312833+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:27:45.312833+0000","last_undegraded":"2023-03-25T04:27:45.312833+0000","last_fullsized":"2023-03-25T04:27:45.312833+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117026,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:29:33 2023","interfaces":[{"interface":"back","average":{"1min":0.26300000000000001,"5min":0.29099999999999998,"15min":0.29599999999999999},"min":{"1min":0.191,"5min":0.191,"15min":0.191},"max":{"1min":0.371,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.20499999999999999},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.29599999999999999,"15min":0.29999999999999999},"min":{"1min":0.21099999999999999,"5min":0.19,"15min":0.19},"max":{"1min":0.39800000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.19500000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051616,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:29:29 2023","interfaces":[{"interface":"back","average":{"1min":0.27200000000000002,"5min":0.253,"15min":0.25},"min":{"1min":0.183,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"last":0.26400000000000001},{"interface":"front","average":{"1min":0.28499999999999998,"5min":0.26800000000000002,"15min":0.26500000000000001},"min":{"1min":0.16600000000000001,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"last":0.29899999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:29:59.629 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean

Ignoring malparsed backtrace: 2023-03-25T04:30:16.889 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:30:16.903 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":170,"stamp":"2023-03-25T04:30:16.128427+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002222"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315468+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:27:45.315468+0000","last_peered":"2023-03-25T04:27:45.315468+0000","last_clean":"2023-03-25T04:27:45.315468+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:27:45.315468+0000","last_undegraded":"2023-03-25T04:27:45.315468+0000","last_fullsized":"2023-03-25T04:27:45.315468+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312845+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:27:45.312845+0000","last_peered":"2023-03-25T04:27:45.312845+0000","last_clean":"2023-03-25T04:27:45.312845+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:27:45.312845+0000","last_undegraded":"2023-03-25T04:27:45.312845+0000","last_fullsized":"2023-03-25T04:27:45.312845+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312882+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:27:45.312882+0000","last_peered":"2023-03-25T04:27:45.312882+0000","last_clean":"2023-03-25T04:27:45.312882+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:27:45.312882+0000","last_undegraded":"2023-03-25T04:27:45.312882+0000","last_fullsized":"2023-03-25T04:27:45.312882+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.311730+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:27:45.311730+0000","last_peered":"2023-03-25T04:27:45.311730+0000","last_clean":"2023-03-25T04:27:45.311730+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:27:45.311730+0000","last_undegraded":"2023-03-25T04:27:45.311730+0000","last_fullsized":"2023-03-25T04:27:45.311730+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315735+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:27:45.315735+0000","last_peered":"2023-03-25T04:27:45.315735+0000","last_clean":"2023-03-25T04:27:45.315735+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:27:45.315735+0000","last_undegraded":"2023-03-25T04:27:45.315735+0000","last_fullsized":"2023-03-25T04:27:45.315735+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312833+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:27:45.312833+0000","last_peered":"2023-03-25T04:27:45.312833+0000","last_clean":"2023-03-25T04:27:45.312833+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:27:45.312833+0000","last_undegraded":"2023-03-25T04:27:45.312833+0000","last_fullsized":"2023-03-25T04:27:45.312833+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117030,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:29:33 2023","interfaces":[{"interface":"back","average":{"1min":0.26300000000000001,"5min":0.29099999999999998,"15min":0.29599999999999999},"min":{"1min":0.191,"5min":0.191,"15min":0.191},"max":{"1min":0.371,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.245},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.29599999999999999,"15min":0.29999999999999999},"min":{"1min":0.21099999999999999,"5min":0.19,"15min":0.19},"max":{"1min":0.39800000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.20899999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051620,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:29:29 2023","interfaces":[{"interface":"back","average":{"1min":0.27200000000000002,"5min":0.253,"15min":0.25},"min":{"1min":0.183,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"last":0.19400000000000001},{"interface":"front","average":{"1min":0.28499999999999998,"5min":0.26800000000000002,"15min":0.26500000000000001},"min":{"1min":0.16600000000000001,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"last":0.23599999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:30:31.984+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:30:32.201+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:30:32.981+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)

Ignoring malparsed backtrace: 2023-03-25T04:30:48.449 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now
, 2023-03-25T04:30:48.450 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:30:48.760 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:31:08.342 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":196,"stamp":"2023-03-25T04:31:08.139018+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002431"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315468+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:27:45.315468+0000","last_peered":"2023-03-25T04:27:45.315468+0000","last_clean":"2023-03-25T04:27:45.315468+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:27:45.315468+0000","last_undegraded":"2023-03-25T04:27:45.315468+0000","last_fullsized":"2023-03-25T04:27:45.315468+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312845+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:27:45.312845+0000","last_peered":"2023-03-25T04:27:45.312845+0000","last_clean":"2023-03-25T04:27:45.312845+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:27:45.312845+0000","last_undegraded":"2023-03-25T04:27:45.312845+0000","last_fullsized":"2023-03-25T04:27:45.312845+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312882+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:27:45.312882+0000","last_peered":"2023-03-25T04:27:45.312882+0000","last_clean":"2023-03-25T04:27:45.312882+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:27:45.312882+0000","last_undegraded":"2023-03-25T04:27:45.312882+0000","last_fullsized":"2023-03-25T04:27:45.312882+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.311730+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:27:45.311730+0000","last_peered":"2023-03-25T04:27:45.311730+0000","last_clean":"2023-03-25T04:27:45.311730+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:27:45.311730+0000","last_undegraded":"2023-03-25T04:27:45.311730+0000","last_fullsized":"2023-03-25T04:27:45.311730+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315735+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:27:45.315735+0000","last_peered":"2023-03-25T04:27:45.315735+0000","last_clean":"2023-03-25T04:27:45.315735+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:27:45.315735+0000","last_undegraded":"2023-03-25T04:27:45.315735+0000","last_fullsized":"2023-03-25T04:27:45.315735+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312833+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:27:45.312833+0000","last_peered":"2023-03-25T04:27:45.312833+0000","last_clean":"2023-03-25T04:27:45.312833+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:27:45.312833+0000","last_undegraded":"2023-03-25T04:27:45.312833+0000","last_fullsized":"2023-03-25T04:27:45.312833+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117040,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:30:37 2023","interfaces":[{"interface":"back","average":{"1min":0.28799999999999998,"5min":0.28899999999999998,"15min":0.29499999999999998},"min":{"1min":0.20200000000000001,"5min":0.191,"15min":0.191},"max":{"1min":0.433,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.29799999999999999},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.28999999999999998,"15min":0.29799999999999999},"min":{"1min":0.17499999999999999,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":0.40300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.25700000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051630,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:30:32 2023","interfaces":[{"interface":"back","average":{"1min":0.25600000000000001,"5min":0.255,"15min":0.251},"min":{"1min":0.19400000000000001,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.35699999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"last":0.22800000000000001},{"interface":"front","average":{"1min":0.28699999999999998,"5min":0.27200000000000002,"15min":0.26600000000000001},"min":{"1min":0.20599999999999999,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.39400000000000002,"5min":0.42899999999999999,"15min":0.42899999999999999},"last":0.24099999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:31:08.343 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace: 2023-03-25T04:31:24.283+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:31:25.192+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:31:25.295+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)

Ignoring malparsed backtrace: 2023-03-25T04:31:40.257 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:31:40.271 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":212,"stamp":"2023-03-25T04:31:40.144732+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002208"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315468+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:27:45.315468+0000","last_peered":"2023-03-25T04:27:45.315468+0000","last_clean":"2023-03-25T04:27:45.315468+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:27:45.315468+0000","last_undegraded":"2023-03-25T04:27:45.315468+0000","last_fullsized":"2023-03-25T04:27:45.315468+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312845+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:27:45.312845+0000","last_peered":"2023-03-25T04:27:45.312845+0000","last_clean":"2023-03-25T04:27:45.312845+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:27:45.312845+0000","last_undegraded":"2023-03-25T04:27:45.312845+0000","last_fullsized":"2023-03-25T04:27:45.312845+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:23.308607+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:29:23.308607+0000","last_peered":"2023-03-25T04:29:23.308607+0000","last_clean":"2023-03-25T04:29:23.308607+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:29:23.308607+0000","last_undegraded":"2023-03-25T04:29:23.308607+0000","last_fullsized":"2023-03-25T04:29:23.308607+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":28,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:20.554710+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:29:20.554710+0000","last_peered":"2023-03-25T04:29:20.554710+0000","last_clean":"2023-03-25T04:29:20.554710+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:29:20.554710+0000","last_undegraded":"2023-03-25T04:29:20.554710+0000","last_fullsized":"2023-03-25T04:29:20.554710+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309216+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:29:28.309216+0000","last_peered":"2023-03-25T04:29:28.309216+0000","last_clean":"2023-03-25T04:29:28.309216+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:29:28.309216+0000","last_undegraded":"2023-03-25T04:29:28.309216+0000","last_fullsized":"2023-03-25T04:29:28.309216+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309233+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:29:28.309233+0000","last_peered":"2023-03-25T04:29:28.309233+0000","last_clean":"2023-03-25T04:29:28.309233+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:29:28.309233+0000","last_undegraded":"2023-03-25T04:29:28.309233+0000","last_fullsized":"2023-03-25T04:29:28.309233+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117046,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:30:37 2023","interfaces":[{"interface":"back","average":{"1min":0.28799999999999998,"5min":0.28899999999999998,"15min":0.29499999999999998},"min":{"1min":0.20200000000000001,"5min":0.191,"15min":0.191},"max":{"1min":0.433,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.216},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.28999999999999998,"15min":0.29799999999999999},"min":{"1min":0.17499999999999999,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":0.40300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.24299999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051637,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:31:35 2023","interfaces":[{"interface":"back","average":{"1min":0.26100000000000001,"5min":0.25700000000000001,"15min":0.251},"min":{"1min":0.19,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.35899999999999999,"5min":0.41599999999999998,"15min":0.41599999999999998},"last":0.221},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.27400000000000002,"15min":0.26700000000000002},"min":{"1min":0.20200000000000001,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.374,"5min":0.42899999999999999,"15min":0.42899999999999999},"last":0.24099999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:32:01.906+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:32:02.388+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:32:02.794 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:32:22.326 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":233,"stamp":"2023-03-25T04:32:22.153289+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002545"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:33.309748+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:29:33.309748+0000","last_peered":"2023-03-25T04:29:33.309748+0000","last_clean":"2023-03-25T04:29:33.309748+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:29:33.309748+0000","last_undegraded":"2023-03-25T04:29:33.309748+0000","last_fullsized":"2023-03-25T04:29:33.309748+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:33.309729+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:29:33.309729+0000","last_peered":"2023-03-25T04:29:33.309729+0000","last_clean":"2023-03-25T04:29:33.309729+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:29:33.309729+0000","last_undegraded":"2023-03-25T04:29:33.309729+0000","last_fullsized":"2023-03-25T04:29:33.309729+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:23.308607+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:29:23.308607+0000","last_peered":"2023-03-25T04:29:23.308607+0000","last_clean":"2023-03-25T04:29:23.308607+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:29:23.308607+0000","last_undegraded":"2023-03-25T04:29:23.308607+0000","last_fullsized":"2023-03-25T04:29:23.308607+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":28,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:20.554710+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:29:20.554710+0000","last_peered":"2023-03-25T04:29:20.554710+0000","last_clean":"2023-03-25T04:29:20.554710+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:29:20.554710+0000","last_undegraded":"2023-03-25T04:29:20.554710+0000","last_fullsized":"2023-03-25T04:29:20.554710+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309216+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:29:28.309216+0000","last_peered":"2023-03-25T04:29:28.309216+0000","last_clean":"2023-03-25T04:29:28.309216+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:29:28.309216+0000","last_undegraded":"2023-03-25T04:29:28.309216+0000","last_fullsized":"2023-03-25T04:29:28.309216+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309233+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:29:28.309233+0000","last_peered":"2023-03-25T04:29:28.309233+0000","last_clean":"2023-03-25T04:29:28.309233+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:29:28.309233+0000","last_undegraded":"2023-03-25T04:29:28.309233+0000","last_fullsized":"2023-03-25T04:29:28.309233+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117055,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:31:42 2023","interfaces":[{"interface":"back","average":{"1min":0.27500000000000002,"5min":0.28399999999999997,"15min":0.29399999999999998},"min":{"1min":0.19800000000000001,"5min":0.191,"15min":0.191},"max":{"1min":0.36899999999999999,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.32300000000000001},{"interface":"front","average":{"1min":0.27900000000000003,"5min":0.28499999999999998,"15min":0.29599999999999999},"min":{"1min":0.216,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":0.40799999999999997,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.27100000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051646,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:31:35 2023","interfaces":[{"interface":"back","average":{"1min":0.26100000000000001,"5min":0.25700000000000001,"15min":0.251},"min":{"1min":0.19,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.35899999999999999,"5min":0.41599999999999998,"15min":0.41599999999999998},"last":0.25600000000000001},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.27400000000000002,"15min":0.26700000000000002},"min":{"1min":0.20200000000000001,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.374,"5min":0.42899999999999999,"15min":0.42899999999999999},"last":0.29199999999999998}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:32:22.327 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean
, 2023-03-25T04:32:22.327 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}

Ignoring malparsed backtrace: 2023-03-25T04:32:39.836 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:32:39.849 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":241,"stamp":"2023-03-25T04:32:38.156312+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002346"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:33.309748+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:29:33.309748+0000","last_peered":"2023-03-25T04:29:33.309748+0000","last_clean":"2023-03-25T04:29:33.309748+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:29:33.309748+0000","last_undegraded":"2023-03-25T04:29:33.309748+0000","last_fullsized":"2023-03-25T04:29:33.309748+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:33.309729+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:29:33.309729+0000","last_peered":"2023-03-25T04:29:33.309729+0000","last_clean":"2023-03-25T04:29:33.309729+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:29:33.309729+0000","last_undegraded":"2023-03-25T04:29:33.309729+0000","last_fullsized":"2023-03-25T04:29:33.309729+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:23.308607+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:29:23.308607+0000","last_peered":"2023-03-25T04:29:23.308607+0000","last_clean":"2023-03-25T04:29:23.308607+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:29:23.308607+0000","last_undegraded":"2023-03-25T04:29:23.308607+0000","last_fullsized":"2023-03-25T04:29:23.308607+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":28,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:20.554710+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:29:20.554710+0000","last_peered":"2023-03-25T04:29:20.554710+0000","last_clean":"2023-03-25T04:29:20.554710+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:29:20.554710+0000","last_undegraded":"2023-03-25T04:29:20.554710+0000","last_fullsized":"2023-03-25T04:29:20.554710+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309216+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:29:28.309216+0000","last_peered":"2023-03-25T04:29:28.309216+0000","last_clean":"2023-03-25T04:29:28.309216+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:29:28.309216+0000","last_undegraded":"2023-03-25T04:29:28.309216+0000","last_fullsized":"2023-03-25T04:29:28.309216+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309233+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:29:28.309233+0000","last_peered":"2023-03-25T04:29:28.309233+0000","last_clean":"2023-03-25T04:29:28.309233+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:29:28.309233+0000","last_undegraded":"2023-03-25T04:29:28.309233+0000","last_fullsized":"2023-03-25T04:29:28.309233+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117058,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:31:42 2023","interfaces":[{"interface":"back","average":{"1min":0.27500000000000002,"5min":0.28399999999999997,"15min":0.29399999999999998},"min":{"1min":0.19800000000000001,"5min":0.191,"15min":0.191},"max":{"1min":0.36899999999999999,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.30599999999999999},{"interface":"front","average":{"1min":0.27900000000000003,"5min":0.28499999999999998,"15min":0.29599999999999999},"min":{"1min":0.216,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":0.40799999999999997,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.32500000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051649,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:31:35 2023","interfaces":[{"interface":"back","average":{"1min":0.26100000000000001,"5min":0.25700000000000001,"15min":0.251},"min":{"1min":0.19,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.35899999999999999,"5min":0.41599999999999998,"15min":0.41599999999999998},"last":0.24199999999999999},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.27400000000000002,"15min":0.26700000000000002},"min":{"1min":0.20200000000000001,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.374,"5min":0.42899999999999999,"15min":0.42899999999999999},"last":0.22600000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:32:54.944+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:32:55.469+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:32:55.927+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)

Ignoring malparsed backtrace: 2023-03-25T04:33:13.958 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean
, 2023-03-25T04:33:13.959 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}
, 2023-03-25T04:33:13.959 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json

Ignoring malparsed backtrace: 2023-03-25T04:33:31.199 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":267,"stamp":"2023-03-25T04:33:30.166660+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002269"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:33.309748+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:29:33.309748+0000","last_peered":"2023-03-25T04:29:33.309748+0000","last_clean":"2023-03-25T04:29:33.309748+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:29:33.309748+0000","last_undegraded":"2023-03-25T04:29:33.309748+0000","last_fullsized":"2023-03-25T04:29:33.309748+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:33.309729+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:29:33.309729+0000","last_peered":"2023-03-25T04:29:33.309729+0000","last_clean":"2023-03-25T04:29:33.309729+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:29:33.309729+0000","last_undegraded":"2023-03-25T04:29:33.309729+0000","last_fullsized":"2023-03-25T04:29:33.309729+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":28,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:31:23.320894+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:31:23.320894+0000","last_peered":"2023-03-25T04:31:23.320894+0000","last_clean":"2023-03-25T04:31:23.320894+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:31:23.320894+0000","last_undegraded":"2023-03-25T04:31:23.320894+0000","last_fullsized":"2023-03-25T04:31:23.320894+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":28,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:20.554710+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:29:20.554710+0000","last_peered":"2023-03-25T04:29:20.554710+0000","last_clean":"2023-03-25T04:29:20.554710+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:29:20.554710+0000","last_undegraded":"2023-03-25T04:29:20.554710+0000","last_fullsized":"2023-03-25T04:29:20.554710+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309216+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:29:28.309216+0000","last_peered":"2023-03-25T04:29:28.309216+0000","last_clean":"2023-03-25T04:29:28.309216+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:29:28.309216+0000","last_undegraded":"2023-03-25T04:29:28.309216+0000","last_fullsized":"2023-03-25T04:29:28.309216+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309233+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:29:28.309233+0000","last_peered":"2023-03-25T04:29:28.309233+0000","last_clean":"2023-03-25T04:29:28.309233+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:29:28.309233+0000","last_undegraded":"2023-03-25T04:29:28.309233+0000","last_fullsized":"2023-03-25T04:29:28.309233+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117068,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:32:46 2023","interfaces":[{"interface":"back","average":{"1min":0.27000000000000002,"5min":0.27900000000000003,"15min":0.29199999999999998},"min":{"1min":0.17899999999999999,"5min":0.17899999999999999,"15min":0.17899999999999999},"max":{"1min":0.35599999999999998,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.28799999999999998},{"interface":"front","average":{"1min":0.26800000000000002,"5min":0.27800000000000002,"15min":0.29399999999999998},"min":{"1min":0.19900000000000001,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":0.377,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.315}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051660,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:32:36 2023","interfaces":[{"interface":"back","average":{"1min":0.248,"5min":0.25700000000000001,"15min":0.251},"min":{"1min":0.17000000000000001,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"last":0.22700000000000001},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.27600000000000002,"15min":0.26800000000000002},"min":{"1min":0.192,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"last":0.20399999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:33:31.201 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:33:31.400+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)

Ignoring malparsed backtrace: 2023-03-25T04:33:47.531+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:33:47.951+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:33:48.505+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)

Ignoring malparsed backtrace: 2023-03-25T04:34:06.104 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json
, 2023-03-25T04:34:06.118+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:34:06.497 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:34:23.399 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:34:23.665+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:34:23.712 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:34:40.195+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:34:40.523+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:34:40.676 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace: 2023-03-25T04:35:03.381+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:35:04.139+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:35:04.391+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)

Ignoring malparsed backtrace: 2023-03-25T04:35:22.738 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean
, 2023-03-25T04:35:22.739 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}
, 2023-03-25T04:35:22.739 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json

Ignoring malparsed backtrace: 2023-03-25T04:35:41.393+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:35:42.352+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:35:42.376+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:35:57.384 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":340,"stamp":"2023-03-25T04:35:56.194992+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.001930"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":36,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:31:43.322894+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:31:43.322894+0000","last_peered":"2023-03-25T04:31:43.322894+0000","last_clean":"2023-03-25T04:31:43.322894+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:31:43.322894+0000","last_undegraded":"2023-03-25T04:31:43.322894+0000","last_fullsized":"2023-03-25T04:31:43.322894+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":28,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:31:43.322876+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:31:43.322876+0000","last_peered":"2023-03-25T04:31:43.322876+0000","last_clean":"2023-03-25T04:31:43.322876+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:31:43.322876+0000","last_undegraded":"2023-03-25T04:31:43.322876+0000","last_fullsized":"2023-03-25T04:31:43.322876+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:23.331998+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:33:23.331998+0000","last_peered":"2023-03-25T04:33:23.331998+0000","last_clean":"2023-03-25T04:33:23.331998+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:33:23.331998+0000","last_undegraded":"2023-03-25T04:33:23.331998+0000","last_fullsized":"2023-03-25T04:33:23.331998+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:30.576723+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:33:30.576723+0000","last_peered":"2023-03-25T04:33:30.576723+0000","last_clean":"2023-03-25T04:33:30.576723+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:33:30.576723+0000","last_undegraded":"2023-03-25T04:33:30.576723+0000","last_fullsized":"2023-03-25T04:33:30.576723+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":37,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:38.333447+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:33:38.333447+0000","last_peered":"2023-03-25T04:33:38.333447+0000","last_clean":"2023-03-25T04:33:38.333447+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:33:38.333447+0000","last_undegraded":"2023-03-25T04:33:38.333447+0000","last_fullsized":"2023-03-25T04:33:38.333447+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:38.333467+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:33:38.333467+0000","last_peered":"2023-03-25T04:33:38.333467+0000","last_clean":"2023-03-25T04:33:38.333467+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:33:38.333467+0000","last_undegraded":"2023-03-25T04:33:38.333467+0000","last_fullsized":"2023-03-25T04:33:38.333467+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117098,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:35:49 2023","interfaces":[{"interface":"back","average":{"1min":0.255,"5min":0.26000000000000001,"15min":0.28199999999999997},"min":{"1min":0.17899999999999999,"5min":0.16,"15min":0.16},"max":{"1min":0.36399999999999999,"5min":0.42599999999999999,"15min":0.45100000000000001},"last":0.26100000000000001},{"interface":"front","average":{"1min":0.26000000000000001,"5min":0.26700000000000002,"15min":0.28599999999999998},"min":{"1min":0.188,"5min":0.17999999999999999,"15min":0.17499999999999999},"max":{"1min":0.39100000000000001,"5min":0.45200000000000001,"15min":0.46300000000000002},"last":0.28399999999999997}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051689,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:35:42 2023","interfaces":[{"interface":"back","average":{"1min":0.29399999999999998,"5min":0.27200000000000002,"15min":0.25800000000000001},"min":{"1min":0.17299999999999999,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.41099999999999998,"5min":0.42699999999999999,"15min":0.42699999999999999},"last":0.20399999999999999},{"interface":"front","average":{"1min":0.30299999999999999,"5min":0.28399999999999997,"15min":0.27300000000000002},"min":{"1min":0.19700000000000001,"5min":0.192,"15min":0.16600000000000001},"max":{"1min":0.43099999999999999,"5min":0.45800000000000002,"15min":0.45800000000000002},"last":0.22}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:35:57.624+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)

Ignoring malparsed backtrace: 2023-03-25T04:36:17.226 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:36:17.239 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":350,"stamp":"2023-03-25T04:36:16.198866+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002148"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":37,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:53.335026+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:33:53.335026+0000","last_peered":"2023-03-25T04:33:53.335026+0000","last_clean":"2023-03-25T04:33:53.335026+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:33:53.335026+0000","last_undegraded":"2023-03-25T04:33:53.335026+0000","last_fullsized":"2023-03-25T04:33:53.335026+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:53.335009+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:33:53.335009+0000","last_peered":"2023-03-25T04:33:53.335009+0000","last_clean":"2023-03-25T04:33:53.335009+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:33:53.335009+0000","last_undegraded":"2023-03-25T04:33:53.335009+0000","last_fullsized":"2023-03-25T04:33:53.335009+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:23.331998+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:33:23.331998+0000","last_peered":"2023-03-25T04:33:23.331998+0000","last_clean":"2023-03-25T04:33:23.331998+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:33:23.331998+0000","last_undegraded":"2023-03-25T04:33:23.331998+0000","last_fullsized":"2023-03-25T04:33:23.331998+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:30.576723+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:33:30.576723+0000","last_peered":"2023-03-25T04:33:30.576723+0000","last_clean":"2023-03-25T04:33:30.576723+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:33:30.576723+0000","last_undegraded":"2023-03-25T04:33:30.576723+0000","last_fullsized":"2023-03-25T04:33:30.576723+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":37,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:38.333447+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:33:38.333447+0000","last_peered":"2023-03-25T04:33:38.333447+0000","last_clean":"2023-03-25T04:33:38.333447+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:33:38.333447+0000","last_undegraded":"2023-03-25T04:33:38.333447+0000","last_fullsized":"2023-03-25T04:33:38.333447+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:38.333467+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:33:38.333467+0000","last_peered":"2023-03-25T04:33:38.333467+0000","last_clean":"2023-03-25T04:33:38.333467+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:33:38.333467+0000","last_undegraded":"2023-03-25T04:33:38.333467+0000","last_fullsized":"2023-03-25T04:33:38.333467+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117102,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:35:49 2023","interfaces":[{"interface":"back","average":{"1min":0.255,"5min":0.26000000000000001,"15min":0.28199999999999997},"min":{"1min":0.17899999999999999,"5min":0.16,"15min":0.16},"max":{"1min":0.36399999999999999,"5min":0.42599999999999999,"15min":0.45100000000000001},"last":0.28399999999999997},{"interface":"front","average":{"1min":0.26000000000000001,"5min":0.26700000000000002,"15min":0.28599999999999998},"min":{"1min":0.188,"5min":0.17999999999999999,"15min":0.17499999999999999},"max":{"1min":0.39100000000000001,"5min":0.45200000000000001,"15min":0.46300000000000002},"last":0.30299999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051693,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:35:42 2023","interfaces":[{"interface":"back","average":{"1min":0.29399999999999998,"5min":0.27200000000000002,"15min":0.25800000000000001},"min":{"1min":0.17299999999999999,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.41099999999999998,"5min":0.42699999999999999,"15min":0.42699999999999999},"last":0.35099999999999998},{"interface":"front","average":{"1min":0.30299999999999999,"5min":0.28399999999999997,"15min":0.27300000000000002},"min":{"1min":0.19700000000000001,"5min":0.192,"15min":0.16600000000000001},"max":{"1min":0.43099999999999999,"5min":0.45800000000000002,"15min":0.45800000000000002},"last":0.32900000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:36:34.556+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:36:34.717+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:36:35.583+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)

Ignoring malparsed backtrace: 2023-03-25T04:36:52.680 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json
, 2023-03-25T04:36:52.701+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:36:53.067 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:37:09.963 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:37:10.273 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all

Ignoring malparsed backtrace: 2023-03-25T04:37:26.856+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:37:27.288 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:37:27.538+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:37:41.903 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":392,"stamp":"2023-03-25T04:37:40.213373+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002012"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":37,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:53.335026+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:33:53.335026+0000","last_peered":"2023-03-25T04:33:53.335026+0000","last_clean":"2023-03-25T04:33:53.335026+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:33:53.335026+0000","last_undegraded":"2023-03-25T04:33:53.335026+0000","last_fullsized":"2023-03-25T04:33:53.335026+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:53.335009+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:33:53.335009+0000","last_peered":"2023-03-25T04:33:53.335009+0000","last_clean":"2023-03-25T04:33:53.335009+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:33:53.335009+0000","last_undegraded":"2023-03-25T04:33:53.335009+0000","last_fullsized":"2023-03-25T04:33:53.335009+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:23.343585+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:35:23.343585+0000","last_peered":"2023-03-25T04:35:23.343585+0000","last_clean":"2023-03-25T04:35:23.343585+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:35:23.343585+0000","last_undegraded":"2023-03-25T04:35:23.343585+0000","last_fullsized":"2023-03-25T04:35:23.343585+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:30.576723+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:33:30.576723+0000","last_peered":"2023-03-25T04:33:30.576723+0000","last_clean":"2023-03-25T04:33:30.576723+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:33:30.576723+0000","last_undegraded":"2023-03-25T04:33:30.576723+0000","last_fullsized":"2023-03-25T04:33:30.576723+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":37,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:38.333447+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:33:38.333447+0000","last_peered":"2023-03-25T04:33:38.333447+0000","last_clean":"2023-03-25T04:33:38.333447+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:33:38.333447+0000","last_undegraded":"2023-03-25T04:33:38.333447+0000","last_fullsized":"2023-03-25T04:33:38.333447+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:38.333467+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:33:38.333467+0000","last_peered":"2023-03-25T04:33:38.333467+0000","last_clean":"2023-03-25T04:33:38.333467+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:33:38.333467+0000","last_undegraded":"2023-03-25T04:33:38.333467+0000","last_fullsized":"2023-03-25T04:33:38.333467+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117119,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:36:50 2023","interfaces":[{"interface":"back","average":{"1min":0.254,"5min":0.25600000000000001,"15min":0.27900000000000003},"min":{"1min":0.20499999999999999,"5min":0.16,"15min":0.16},"max":{"1min":0.371,"5min":0.42599999999999999,"15min":0.45100000000000001},"last":0.23000000000000001},{"interface":"front","average":{"1min":0.26800000000000002,"5min":0.26500000000000001,"15min":0.28399999999999997},"min":{"1min":0.19700000000000001,"5min":0.17999999999999999,"15min":0.17499999999999999},"max":{"1min":0.34699999999999998,"5min":0.45200000000000001,"15min":0.46300000000000002},"last":0.20899999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051710,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:36:44 2023","interfaces":[{"interface":"back","average":{"1min":0.28399999999999997,"5min":0.27700000000000002,"15min":0.26100000000000001},"min":{"1min":0.192,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.40899999999999997,"5min":0.42699999999999999,"15min":0.42699999999999999},"last":0.39700000000000002},{"interface":"front","average":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.27500000000000002},"min":{"1min":0.182,"5min":0.182,"15min":0.16600000000000001},"max":{"1min":0.439,"5min":0.45800000000000002,"15min":0.45800000000000002},"last":0.439}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:37:42.400+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)

Ignoring malparsed backtrace: 2023-03-25T04:38:04.766+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:38:04.922 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:38:05.246 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:38:19.605 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":411,"stamp":"2023-03-25T04:38:18.220966+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002368"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":38,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:36:03.347070+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:36:03.347070+0000","last_peered":"2023-03-25T04:36:03.347070+0000","last_clean":"2023-03-25T04:36:03.347070+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:36:03.347070+0000","last_undegraded":"2023-03-25T04:36:03.347070+0000","last_fullsized":"2023-03-25T04:36:03.347070+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:36:03.347025+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:36:03.347025+0000","last_peered":"2023-03-25T04:36:03.347025+0000","last_clean":"2023-03-25T04:36:03.347025+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:36:03.347025+0000","last_undegraded":"2023-03-25T04:36:03.347025+0000","last_fullsized":"2023-03-25T04:36:03.347025+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:23.343585+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:35:23.343585+0000","last_peered":"2023-03-25T04:35:23.343585+0000","last_clean":"2023-03-25T04:35:23.343585+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:35:23.343585+0000","last_undegraded":"2023-03-25T04:35:23.343585+0000","last_fullsized":"2023-03-25T04:35:23.343585+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:35.587025+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:35:35.587025+0000","last_peered":"2023-03-25T04:35:35.587025+0000","last_clean":"2023-03-25T04:35:35.587025+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:35:35.587025+0000","last_undegraded":"2023-03-25T04:35:35.587025+0000","last_fullsized":"2023-03-25T04:35:35.587025+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":38,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:43.345308+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:35:43.345308+0000","last_peered":"2023-03-25T04:35:43.345308+0000","last_clean":"2023-03-25T04:35:43.345308+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:35:43.345308+0000","last_undegraded":"2023-03-25T04:35:43.345308+0000","last_fullsized":"2023-03-25T04:35:43.345308+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:43.345324+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:35:43.345324+0000","last_peered":"2023-03-25T04:35:43.345324+0000","last_clean":"2023-03-25T04:35:43.345324+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:35:43.345324+0000","last_undegraded":"2023-03-25T04:35:43.345324+0000","last_fullsized":"2023-03-25T04:35:43.345324+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117127,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:37:53 2023","interfaces":[{"interface":"back","average":{"1min":0.25800000000000001,"5min":0.253,"15min":0.27700000000000002},"min":{"1min":0.184,"5min":0.16,"15min":0.16},"max":{"1min":0.44600000000000001,"5min":0.44600000000000001,"15min":0.45100000000000001},"last":0.35199999999999998},{"interface":"front","average":{"1min":0.26900000000000002,"5min":0.26500000000000001,"15min":0.28199999999999997},"min":{"1min":0.192,"5min":0.17999999999999999,"15min":0.17499999999999999},"max":{"1min":0.46400000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"last":0.33200000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051717,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:37:44 2023","interfaces":[{"interface":"back","average":{"1min":0.27300000000000002,"5min":0.28199999999999997,"15min":0.26200000000000001},"min":{"1min":0.17899999999999999,"5min":0.17299999999999999,"15min":0.17000000000000001},"max":{"1min":0.41799999999999998,"5min":0.42099999999999999,"15min":0.42699999999999999},"last":0.28699999999999998},{"interface":"front","average":{"1min":0.29399999999999998,"5min":0.29099999999999998,"15min":0.27700000000000002},"min":{"1min":0.19900000000000001,"5min":0.182,"15min":0.16600000000000001},"max":{"1min":0.441,"5min":0.441,"15min":0.45800000000000002},"last":0.26300000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:38:19.718+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:38:20.315+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)

Ignoring malparsed backtrace: 2023-03-25T04:38:44.283 INFO:teuthology.orchestra.run.smithi070.stdout:
, 2023-03-25T04:38:44.283 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":781,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":16,"modified":"2023-03-25T04:38:40.225321+0000","services":{}},"progress_events":{}}
, 2023-03-25T04:38:44.296 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now

Ignoring malparsed backtrace: 2023-03-25T04:39:02.477+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:39:02.578+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:39:03.524+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:39:20.359 INFO:teuthology.orchestra.run.smithi070.stdout:
, 2023-03-25T04:39:20.359 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":818,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":16,"modified":"2023-03-25T04:38:40.225321+0000","services":{}},"progress_events":{}}
, 2023-03-25T04:39:20.372 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now

Ignoring malparsed backtrace: 2023-03-25T04:39:37.527 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:39:37.541 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":450,"stamp":"2023-03-25T04:39:36.235332+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002086"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":38,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:36:03.347070+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:36:03.347070+0000","last_peered":"2023-03-25T04:36:03.347070+0000","last_clean":"2023-03-25T04:36:03.347070+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:36:03.347070+0000","last_undegraded":"2023-03-25T04:36:03.347070+0000","last_fullsized":"2023-03-25T04:36:03.347070+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:36:03.347025+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:36:03.347025+0000","last_peered":"2023-03-25T04:36:03.347025+0000","last_clean":"2023-03-25T04:36:03.347025+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:36:03.347025+0000","last_undegraded":"2023-03-25T04:36:03.347025+0000","last_fullsized":"2023-03-25T04:36:03.347025+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:23.353778+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:37:23.353778+0000","last_peered":"2023-03-25T04:37:23.353778+0000","last_clean":"2023-03-25T04:37:23.353778+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:37:23.353778+0000","last_undegraded":"2023-03-25T04:37:23.353778+0000","last_fullsized":"2023-03-25T04:37:23.353778+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:35.587025+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:35:35.587025+0000","last_peered":"2023-03-25T04:35:35.587025+0000","last_clean":"2023-03-25T04:35:35.587025+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:35:35.587025+0000","last_undegraded":"2023-03-25T04:35:35.587025+0000","last_fullsized":"2023-03-25T04:35:35.587025+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":38,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:43.345308+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:35:43.345308+0000","last_peered":"2023-03-25T04:35:43.345308+0000","last_clean":"2023-03-25T04:35:43.345308+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:35:43.345308+0000","last_undegraded":"2023-03-25T04:35:43.345308+0000","last_fullsized":"2023-03-25T04:35:43.345308+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:43.345324+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:35:43.345324+0000","last_peered":"2023-03-25T04:35:43.345324+0000","last_clean":"2023-03-25T04:35:43.345324+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:35:43.345324+0000","last_undegraded":"2023-03-25T04:35:43.345324+0000","last_fullsized":"2023-03-25T04:35:43.345324+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117143,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:38:54 2023","interfaces":[{"interface":"back","average":{"1min":0.25800000000000001,"5min":0.25700000000000001,"15min":0.27400000000000002},"min":{"1min":0.192,"5min":0.17399999999999999,"15min":0.16},"max":{"1min":0.39400000000000002,"5min":0.44600000000000001,"15min":0.45100000000000001},"last":0.26500000000000001},{"interface":"front","average":{"1min":0.25900000000000001,"5min":0.26700000000000002,"15min":0.27900000000000003},"min":{"1min":0.19500000000000001,"5min":0.188,"15min":0.17499999999999999},"max":{"1min":0.41499999999999998,"5min":0.46400000000000002,"15min":0.46400000000000002},"last":0.29099999999999998}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051733,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:38:45 2023","interfaces":[{"interface":"back","average":{"1min":0.27300000000000002,"5min":0.28100000000000003,"15min":0.26400000000000001},"min":{"1min":0.19400000000000001,"5min":0.17299999999999999,"15min":0.17000000000000001},"max":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"last":0.27300000000000002},{"interface":"front","average":{"1min":0.28799999999999998,"5min":0.29099999999999998,"15min":0.27800000000000002},"min":{"1min":0.19800000000000001,"5min":0.182,"15min":0.16600000000000001},"max":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"last":0.308}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:39:54.416+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:39:54.455 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:39:54.780 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:40:09.288+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:40:09.389+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:40:10.267+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)

Ignoring malparsed backtrace: 2023-03-25T04:40:28.984 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":476,"stamp":"2023-03-25T04:40:28.245911+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002693"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":39,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:38:13.358968+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:38:13.358968+0000","last_peered":"2023-03-25T04:38:13.358968+0000","last_clean":"2023-03-25T04:38:13.358968+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:38:13.358968+0000","last_undegraded":"2023-03-25T04:38:13.358968+0000","last_fullsized":"2023-03-25T04:38:13.358968+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:38:13.358922+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:38:13.358922+0000","last_peered":"2023-03-25T04:38:13.358922+0000","last_clean":"2023-03-25T04:38:13.358922+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:38:13.358922+0000","last_undegraded":"2023-03-25T04:38:13.358922+0000","last_fullsized":"2023-03-25T04:38:13.358922+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:23.353778+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:37:23.353778+0000","last_peered":"2023-03-25T04:37:23.353778+0000","last_clean":"2023-03-25T04:37:23.353778+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:37:23.353778+0000","last_undegraded":"2023-03-25T04:37:23.353778+0000","last_fullsized":"2023-03-25T04:37:23.353778+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:40.598579+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:37:40.598579+0000","last_peered":"2023-03-25T04:37:40.598579+0000","last_clean":"2023-03-25T04:37:40.598579+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:37:40.598579+0000","last_undegraded":"2023-03-25T04:37:40.598579+0000","last_fullsized":"2023-03-25T04:37:40.598579+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":39,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:48.356344+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:37:48.356344+0000","last_peered":"2023-03-25T04:37:48.356344+0000","last_clean":"2023-03-25T04:37:48.356344+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:37:48.356344+0000","last_undegraded":"2023-03-25T04:37:48.356344+0000","last_fullsized":"2023-03-25T04:37:48.356344+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:48.356365+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:37:48.356365+0000","last_peered":"2023-03-25T04:37:48.356365+0000","last_clean":"2023-03-25T04:37:48.356365+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:37:48.356365+0000","last_undegraded":"2023-03-25T04:37:48.356365+0000","last_fullsized":"2023-03-25T04:37:48.356365+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117153,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:39:54 2023","interfaces":[{"interface":"back","average":{"1min":0.25900000000000001,"5min":0.25600000000000001,"15min":0.27100000000000002},"min":{"1min":0.188,"5min":0.17899999999999999,"15min":0.16},"max":{"1min":0.41999999999999998,"5min":0.44600000000000001,"15min":0.45100000000000001},"last":0.255},{"interface":"front","average":{"1min":0.27500000000000002,"5min":0.26600000000000001,"15min":0.27700000000000002},"min":{"1min":0.17699999999999999,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.45600000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"last":0.26800000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051743,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:39:47 2023","interfaces":[{"interface":"back","average":{"1min":0.26300000000000001,"5min":0.27700000000000002,"15min":0.26500000000000001},"min":{"1min":0.187,"5min":0.17299999999999999,"15min":0.17000000000000001},"max":{"1min":0.433,"5min":0.433,"15min":0.433},"last":0.36599999999999999},{"interface":"front","average":{"1min":0.27100000000000002,"5min":0.28799999999999998,"15min":0.27900000000000003},"min":{"1min":0.20100000000000001,"5min":0.182,"15min":0.16600000000000001},"max":{"1min":0.48599999999999999,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.34999999999999998}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:40:28.986 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean
, 2023-03-25T04:40:28.986 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}

Ignoring malparsed backtrace: 2023-03-25T04:40:47.229+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:40:47.503+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:40:48.251+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)

Ignoring malparsed backtrace: 2023-03-25T04:41:03.267 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:41:03.578 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all

Ignoring malparsed backtrace: 2023-03-25T04:41:21.347 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":502,"stamp":"2023-03-25T04:41:20.256204+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002342"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":39,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:38:13.358968+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:38:13.358968+0000","last_peered":"2023-03-25T04:38:13.358968+0000","last_clean":"2023-03-25T04:38:13.358968+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:38:13.358968+0000","last_undegraded":"2023-03-25T04:38:13.358968+0000","last_fullsized":"2023-03-25T04:38:13.358968+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:38:13.358922+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:38:13.358922+0000","last_peered":"2023-03-25T04:38:13.358922+0000","last_clean":"2023-03-25T04:38:13.358922+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:38:13.358922+0000","last_undegraded":"2023-03-25T04:38:13.358922+0000","last_fullsized":"2023-03-25T04:38:13.358922+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:23.353778+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:37:23.353778+0000","last_peered":"2023-03-25T04:37:23.353778+0000","last_clean":"2023-03-25T04:37:23.353778+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:37:23.353778+0000","last_undegraded":"2023-03-25T04:37:23.353778+0000","last_fullsized":"2023-03-25T04:37:23.353778+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:40.598579+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:37:40.598579+0000","last_peered":"2023-03-25T04:37:40.598579+0000","last_clean":"2023-03-25T04:37:40.598579+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:37:40.598579+0000","last_undegraded":"2023-03-25T04:37:40.598579+0000","last_fullsized":"2023-03-25T04:37:40.598579+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":39,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:48.356344+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:37:48.356344+0000","last_peered":"2023-03-25T04:37:48.356344+0000","last_clean":"2023-03-25T04:37:48.356344+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:37:48.356344+0000","last_undegraded":"2023-03-25T04:37:48.356344+0000","last_fullsized":"2023-03-25T04:37:48.356344+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:48.356365+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:37:48.356365+0000","last_peered":"2023-03-25T04:37:48.356365+0000","last_clean":"2023-03-25T04:37:48.356365+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:37:48.356365+0000","last_undegraded":"2023-03-25T04:37:48.356365+0000","last_fullsized":"2023-03-25T04:37:48.356365+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117163,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:40:56 2023","interfaces":[{"interface":"back","average":{"1min":0.28799999999999998,"5min":0.26300000000000001,"15min":0.27100000000000002},"min":{"1min":0.188,"5min":0.184,"15min":0.16},"max":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.17499999999999999},{"interface":"front","average":{"1min":0.29599999999999999,"5min":0.27300000000000002,"15min":0.27700000000000002},"min":{"1min":0.20599999999999999,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.43099999999999999,"5min":0.46400000000000002,"15min":0.46400000000000002},"last":0.20699999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051754,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:40:52 2023","interfaces":[{"interface":"back","average":{"1min":0.28599999999999998,"5min":0.27500000000000002,"15min":0.26700000000000002},"min":{"1min":0.17899999999999999,"5min":0.17899999999999999,"15min":0.17000000000000001},"max":{"1min":0.44,"5min":0.44,"15min":0.44},"last":0.23699999999999999},{"interface":"front","average":{"1min":0.29799999999999999,"5min":0.28699999999999998,"15min":0.28100000000000003},"min":{"1min":0.20300000000000001,"5min":0.182,"15min":0.16600000000000001},"max":{"1min":0.47599999999999998,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.28000000000000003}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:41:21.349 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean
, 2023-03-25T04:41:21.349 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}

Ignoring malparsed backtrace: 2023-03-25T04:41:39.867+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:41:40.383+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:41:40.905+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:41:58.219 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json
, 2023-03-25T04:41:58.319+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:41:58.644 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:42:18.193 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":530,"stamp":"2023-03-25T04:42:16.266659+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002356"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":39,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:38:13.358968+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:38:13.358968+0000","last_peered":"2023-03-25T04:38:13.358968+0000","last_clean":"2023-03-25T04:38:13.358968+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:38:13.358968+0000","last_undegraded":"2023-03-25T04:38:13.358968+0000","last_fullsized":"2023-03-25T04:38:13.358968+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:38:13.358922+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:38:13.358922+0000","last_peered":"2023-03-25T04:38:13.358922+0000","last_clean":"2023-03-25T04:38:13.358922+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:38:13.358922+0000","last_undegraded":"2023-03-25T04:38:13.358922+0000","last_fullsized":"2023-03-25T04:38:13.358922+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:23.364593+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:39:23.364593+0000","last_peered":"2023-03-25T04:39:23.364593+0000","last_clean":"2023-03-25T04:39:23.364593+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:39:23.364593+0000","last_undegraded":"2023-03-25T04:39:23.364593+0000","last_fullsized":"2023-03-25T04:39:23.364593+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:45.608475+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:39:45.608475+0000","last_peered":"2023-03-25T04:39:45.608475+0000","last_clean":"2023-03-25T04:39:45.608475+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:39:45.608475+0000","last_undegraded":"2023-03-25T04:39:45.608475+0000","last_fullsized":"2023-03-25T04:39:45.608475+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":40,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:53.367398+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:39:53.367398+0000","last_peered":"2023-03-25T04:39:53.367398+0000","last_clean":"2023-03-25T04:39:53.367398+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:39:53.367398+0000","last_undegraded":"2023-03-25T04:39:53.367398+0000","last_fullsized":"2023-03-25T04:39:53.367398+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:53.367424+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:39:53.367424+0000","last_peered":"2023-03-25T04:39:53.367424+0000","last_clean":"2023-03-25T04:39:53.367424+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:39:53.367424+0000","last_undegraded":"2023-03-25T04:39:53.367424+0000","last_fullsized":"2023-03-25T04:39:53.367424+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117175,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:41:58 2023","interfaces":[{"interface":"back","average":{"1min":0.25800000000000001,"5min":0.26400000000000001,"15min":0.26800000000000002},"min":{"1min":0.17499999999999999,"5min":0.17499999999999999,"15min":0.16},"max":{"1min":0.39100000000000001,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.22900000000000001},{"interface":"front","average":{"1min":0.26300000000000001,"5min":0.27200000000000002,"15min":0.27400000000000002},"min":{"1min":0.20599999999999999,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.35699999999999998,"5min":0.46400000000000002,"15min":0.46400000000000002},"last":0.26400000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051765,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:41:52 2023","interfaces":[{"interface":"back","average":{"1min":0.26500000000000001,"5min":0.27200000000000002,"15min":0.26800000000000002},"min":{"1min":0.17000000000000001,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.39200000000000002,"5min":0.44,"15min":0.44},"last":0.23899999999999999},{"interface":"front","average":{"1min":0.28100000000000003,"5min":0.28599999999999998,"15min":0.28199999999999997},"min":{"1min":0.19400000000000001,"5min":0.19400000000000001,"15min":0.16600000000000001},"max":{"1min":0.41399999999999998,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.254}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:42:18.297+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:42:18.958+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:42:35.512 INFO:teuthology.orchestra.run.smithi070.stdout:
, 2023-03-25T04:42:35.512 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":1013,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":17,"modified":"2023-03-25T04:41:54.262744+0000","services":{}},"progress_events":{}}
, 2023-03-25T04:42:35.525 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:42:52.741 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":548,"stamp":"2023-03-25T04:42:52.273601+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002221"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":40,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:40:23.370056+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:40:23.370056+0000","last_peered":"2023-03-25T04:40:23.370056+0000","last_clean":"2023-03-25T04:40:23.370056+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:40:23.370056+0000","last_undegraded":"2023-03-25T04:40:23.370056+0000","last_fullsized":"2023-03-25T04:40:23.370056+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:40:23.370039+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:40:23.370039+0000","last_peered":"2023-03-25T04:40:23.370039+0000","last_clean":"2023-03-25T04:40:23.370039+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:40:23.370039+0000","last_undegraded":"2023-03-25T04:40:23.370039+0000","last_fullsized":"2023-03-25T04:40:23.370039+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:23.364593+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:39:23.364593+0000","last_peered":"2023-03-25T04:39:23.364593+0000","last_clean":"2023-03-25T04:39:23.364593+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:39:23.364593+0000","last_undegraded":"2023-03-25T04:39:23.364593+0000","last_fullsized":"2023-03-25T04:39:23.364593+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:45.608475+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:39:45.608475+0000","last_peered":"2023-03-25T04:39:45.608475+0000","last_clean":"2023-03-25T04:39:45.608475+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:39:45.608475+0000","last_undegraded":"2023-03-25T04:39:45.608475+0000","last_fullsized":"2023-03-25T04:39:45.608475+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":40,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:53.367398+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:39:53.367398+0000","last_peered":"2023-03-25T04:39:53.367398+0000","last_clean":"2023-03-25T04:39:53.367398+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:39:53.367398+0000","last_undegraded":"2023-03-25T04:39:53.367398+0000","last_fullsized":"2023-03-25T04:39:53.367398+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:53.367424+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:39:53.367424+0000","last_peered":"2023-03-25T04:39:53.367424+0000","last_clean":"2023-03-25T04:39:53.367424+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:39:53.367424+0000","last_undegraded":"2023-03-25T04:39:53.367424+0000","last_fullsized":"2023-03-25T04:39:53.367424+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117182,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:41:58 2023","interfaces":[{"interface":"back","average":{"1min":0.25800000000000001,"5min":0.26400000000000001,"15min":0.26800000000000002},"min":{"1min":0.17499999999999999,"5min":0.17499999999999999,"15min":0.16},"max":{"1min":0.39100000000000001,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.19600000000000001},{"interface":"front","average":{"1min":0.26300000000000001,"5min":0.27200000000000002,"15min":0.27400000000000002},"min":{"1min":0.20599999999999999,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.35699999999999998,"5min":0.46400000000000002,"15min":0.46400000000000002},"last":0.219}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051773,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:41:52 2023","interfaces":[{"interface":"back","average":{"1min":0.26500000000000001,"5min":0.27200000000000002,"15min":0.26800000000000002},"min":{"1min":0.17000000000000001,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.39200000000000002,"5min":0.44,"15min":0.44},"last":0.19},{"interface":"front","average":{"1min":0.28100000000000003,"5min":0.28599999999999998,"15min":0.28199999999999997},"min":{"1min":0.19400000000000001,"5min":0.19400000000000001,"15min":0.16600000000000001},"max":{"1min":0.41399999999999998,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.17599999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:42:52.742 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean

Ignoring malparsed backtrace: 2023-03-25T04:43:09.973 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:43:09.986 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":556,"stamp":"2023-03-25T04:43:08.276556+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002359"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":40,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:40:23.370056+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:40:23.370056+0000","last_peered":"2023-03-25T04:40:23.370056+0000","last_clean":"2023-03-25T04:40:23.370056+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:40:23.370056+0000","last_undegraded":"2023-03-25T04:40:23.370056+0000","last_fullsized":"2023-03-25T04:40:23.370056+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:40:23.370039+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:40:23.370039+0000","last_peered":"2023-03-25T04:40:23.370039+0000","last_clean":"2023-03-25T04:40:23.370039+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:40:23.370039+0000","last_undegraded":"2023-03-25T04:40:23.370039+0000","last_fullsized":"2023-03-25T04:40:23.370039+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:23.364593+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:39:23.364593+0000","last_peered":"2023-03-25T04:39:23.364593+0000","last_clean":"2023-03-25T04:39:23.364593+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:39:23.364593+0000","last_undegraded":"2023-03-25T04:39:23.364593+0000","last_fullsized":"2023-03-25T04:39:23.364593+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:45.608475+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:39:45.608475+0000","last_peered":"2023-03-25T04:39:45.608475+0000","last_clean":"2023-03-25T04:39:45.608475+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:39:45.608475+0000","last_undegraded":"2023-03-25T04:39:45.608475+0000","last_fullsized":"2023-03-25T04:39:45.608475+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":40,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:53.367398+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:39:53.367398+0000","last_peered":"2023-03-25T04:39:53.367398+0000","last_clean":"2023-03-25T04:39:53.367398+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:39:53.367398+0000","last_undegraded":"2023-03-25T04:39:53.367398+0000","last_fullsized":"2023-03-25T04:39:53.367398+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:53.367424+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:39:53.367424+0000","last_peered":"2023-03-25T04:39:53.367424+0000","last_clean":"2023-03-25T04:39:53.367424+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:39:53.367424+0000","last_undegraded":"2023-03-25T04:39:53.367424+0000","last_fullsized":"2023-03-25T04:39:53.367424+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117185,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:43:00 2023","interfaces":[{"interface":"back","average":{"1min":0.27400000000000002,"5min":0.26700000000000002,"15min":0.26600000000000001},"min":{"1min":0.19600000000000001,"5min":0.17499999999999999,"15min":0.16},"max":{"1min":0.42999999999999999,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.27800000000000002},{"interface":"front","average":{"1min":0.28199999999999997,"5min":0.27500000000000002,"15min":0.27300000000000002},"min":{"1min":0.185,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.47199999999999998,"5min":0.47199999999999998,"15min":0.47199999999999998},"last":0.247}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051776,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:42:54 2023","interfaces":[{"interface":"back","average":{"1min":0.26300000000000001,"5min":0.27000000000000002,"15min":0.26900000000000002},"min":{"1min":0.17399999999999999,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.438,"5min":0.44,"15min":0.44},"last":0.20100000000000001},{"interface":"front","average":{"1min":0.27400000000000002,"5min":0.28199999999999997,"15min":0.28299999999999997},"min":{"1min":0.17599999999999999,"5min":0.17599999999999999,"15min":0.16600000000000001},"max":{"1min":0.41599999999999998,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.23499999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:43:25.042+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:43:25.411+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:43:26.013+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:43:47.937+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:43:48.534+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:43:48.889+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:44:09.318 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:44:09.331 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":586,"stamp":"2023-03-25T04:44:08.288933+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002697"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":40,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:40:23.370056+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:40:23.370056+0000","last_peered":"2023-03-25T04:40:23.370056+0000","last_clean":"2023-03-25T04:40:23.370056+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:40:23.370056+0000","last_undegraded":"2023-03-25T04:40:23.370056+0000","last_fullsized":"2023-03-25T04:40:23.370056+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:40:23.370039+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:40:23.370039+0000","last_peered":"2023-03-25T04:40:23.370039+0000","last_clean":"2023-03-25T04:40:23.370039+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:40:23.370039+0000","last_undegraded":"2023-03-25T04:40:23.370039+0000","last_fullsized":"2023-03-25T04:40:23.370039+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:23.375106+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:41:23.375106+0000","last_peered":"2023-03-25T04:41:23.375106+0000","last_clean":"2023-03-25T04:41:23.375106+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:41:23.375106+0000","last_undegraded":"2023-03-25T04:41:23.375106+0000","last_fullsized":"2023-03-25T04:41:23.375106+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:50.618930+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:41:50.618930+0000","last_peered":"2023-03-25T04:41:50.618930+0000","last_clean":"2023-03-25T04:41:50.618930+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:41:50.618930+0000","last_undegraded":"2023-03-25T04:41:50.618930+0000","last_fullsized":"2023-03-25T04:41:50.618930+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":41,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:58.377821+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:41:58.377821+0000","last_peered":"2023-03-25T04:41:58.377821+0000","last_clean":"2023-03-25T04:41:58.377821+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:41:58.377821+0000","last_undegraded":"2023-03-25T04:41:58.377821+0000","last_fullsized":"2023-03-25T04:41:58.377821+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:58.377836+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:41:58.377836+0000","last_peered":"2023-03-25T04:41:58.377836+0000","last_clean":"2023-03-25T04:41:58.377836+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:41:58.377836+0000","last_undegraded":"2023-03-25T04:41:58.377836+0000","last_fullsized":"2023-03-25T04:41:58.377836+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117197,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:44:01 2023","interfaces":[{"interface":"back","average":{"1min":0.27200000000000002,"5min":0.27000000000000002,"15min":0.26400000000000001},"min":{"1min":0.20399999999999999,"5min":0.17499999999999999,"15min":0.16},"max":{"1min":0.38400000000000001,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.245},{"interface":"front","average":{"1min":0.27600000000000002,"5min":0.27800000000000002,"15min":0.27100000000000002},"min":{"1min":0.23100000000000001,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.40400000000000003,"5min":0.47199999999999998,"15min":0.47199999999999998},"last":0.27500000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051788,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:43:55 2023","interfaces":[{"interface":"back","average":{"1min":0.26100000000000001,"5min":0.26700000000000002,"15min":0.27000000000000002},"min":{"1min":0.17799999999999999,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.374,"5min":0.44,"15min":0.44},"last":0.30299999999999999},{"interface":"front","average":{"1min":0.28299999999999997,"5min":0.28100000000000003,"15min":0.28399999999999997},"min":{"1min":0.20899999999999999,"5min":0.17599999999999999,"15min":0.16600000000000001},"max":{"1min":0.40100000000000002,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.27800000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:44:28.923 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":1126,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":19,"modified":"2023-03-25T04:43:20.278959+0000","services":{}},"progress_events":{}}
, 2023-03-25T04:44:28.936 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now
, 2023-03-25T04:44:28.937 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace: 2023-03-25T04:44:48.786 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:44:48.800 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":606,"stamp":"2023-03-25T04:44:48.296805+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002182"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":41,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:42:34.382196+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:42:34.382196+0000","last_peered":"2023-03-25T04:42:34.382196+0000","last_clean":"2023-03-25T04:42:34.382196+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:42:34.382196+0000","last_undegraded":"2023-03-25T04:42:34.382196+0000","last_fullsized":"2023-03-25T04:42:34.382196+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:42:29.381744+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:42:29.381744+0000","last_peered":"2023-03-25T04:42:29.381744+0000","last_clean":"2023-03-25T04:42:29.381744+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:42:29.381744+0000","last_undegraded":"2023-03-25T04:42:29.381744+0000","last_fullsized":"2023-03-25T04:42:29.381744+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:23.375106+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:41:23.375106+0000","last_peered":"2023-03-25T04:41:23.375106+0000","last_clean":"2023-03-25T04:41:23.375106+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:41:23.375106+0000","last_undegraded":"2023-03-25T04:41:23.375106+0000","last_fullsized":"2023-03-25T04:41:23.375106+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:50.618930+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:41:50.618930+0000","last_peered":"2023-03-25T04:41:50.618930+0000","last_clean":"2023-03-25T04:41:50.618930+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:41:50.618930+0000","last_undegraded":"2023-03-25T04:41:50.618930+0000","last_fullsized":"2023-03-25T04:41:50.618930+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":41,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:58.377821+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:41:58.377821+0000","last_peered":"2023-03-25T04:41:58.377821+0000","last_clean":"2023-03-25T04:41:58.377821+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:41:58.377821+0000","last_undegraded":"2023-03-25T04:41:58.377821+0000","last_fullsized":"2023-03-25T04:41:58.377821+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:58.377836+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:41:58.377836+0000","last_peered":"2023-03-25T04:41:58.377836+0000","last_clean":"2023-03-25T04:41:58.377836+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:41:58.377836+0000","last_undegraded":"2023-03-25T04:41:58.377836+0000","last_fullsized":"2023-03-25T04:41:58.377836+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117206,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:44:01 2023","interfaces":[{"interface":"back","average":{"1min":0.27200000000000002,"5min":0.27000000000000002,"15min":0.26400000000000001},"min":{"1min":0.20399999999999999,"5min":0.17499999999999999,"15min":0.16},"max":{"1min":0.38400000000000001,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.216},{"interface":"front","average":{"1min":0.27600000000000002,"5min":0.27800000000000002,"15min":0.27100000000000002},"min":{"1min":0.23100000000000001,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.40400000000000003,"5min":0.47199999999999998,"15min":0.47199999999999998},"last":0.23999999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051796,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:43:55 2023","interfaces":[{"interface":"back","average":{"1min":0.26100000000000001,"5min":0.26700000000000002,"15min":0.27000000000000002},"min":{"1min":0.17799999999999999,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.374,"5min":0.44,"15min":0.44},"last":0.25600000000000001},{"interface":"front","average":{"1min":0.28299999999999997,"5min":0.28100000000000003,"15min":0.28399999999999997},"min":{"1min":0.20899999999999999,"5min":0.17599999999999999,"15min":0.16600000000000001},"max":{"1min":0.40100000000000002,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.28000000000000003}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:45:05.762 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:45:06.073 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all

Ignoring malparsed backtrace: 2023-03-25T04:45:25.658 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":624,"stamp":"2023-03-25T04:45:24.304294+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002366"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":41,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:42:34.382196+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:42:34.382196+0000","last_peered":"2023-03-25T04:42:34.382196+0000","last_clean":"2023-03-25T04:42:34.382196+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:42:34.382196+0000","last_undegraded":"2023-03-25T04:42:34.382196+0000","last_fullsized":"2023-03-25T04:42:34.382196+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:42:29.381744+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:42:29.381744+0000","last_peered":"2023-03-25T04:42:29.381744+0000","last_clean":"2023-03-25T04:42:29.381744+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:42:29.381744+0000","last_undegraded":"2023-03-25T04:42:29.381744+0000","last_fullsized":"2023-03-25T04:42:29.381744+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:23.375106+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:41:23.375106+0000","last_peered":"2023-03-25T04:41:23.375106+0000","last_clean":"2023-03-25T04:41:23.375106+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:41:23.375106+0000","last_undegraded":"2023-03-25T04:41:23.375106+0000","last_fullsized":"2023-03-25T04:41:23.375106+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:50.618930+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:41:50.618930+0000","last_peered":"2023-03-25T04:41:50.618930+0000","last_clean":"2023-03-25T04:41:50.618930+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:41:50.618930+0000","last_undegraded":"2023-03-25T04:41:50.618930+0000","last_fullsized":"2023-03-25T04:41:50.618930+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":41,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:58.377821+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:41:58.377821+0000","last_peered":"2023-03-25T04:41:58.377821+0000","last_clean":"2023-03-25T04:41:58.377821+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:41:58.377821+0000","last_undegraded":"2023-03-25T04:41:58.377821+0000","last_fullsized":"2023-03-25T04:41:58.377821+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:58.377836+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:41:58.377836+0000","last_peered":"2023-03-25T04:41:58.377836+0000","last_clean":"2023-03-25T04:41:58.377836+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:41:58.377836+0000","last_undegraded":"2023-03-25T04:41:58.377836+0000","last_fullsized":"2023-03-25T04:41:58.377836+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117213,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:45:01 2023","interfaces":[{"interface":"back","average":{"1min":0.251,"5min":0.26800000000000002,"15min":0.26400000000000001},"min":{"1min":0.183,"5min":0.17499999999999999,"15min":0.16},"max":{"1min":0.41499999999999998,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.26900000000000002},{"interface":"front","average":{"1min":0.28000000000000003,"5min":0.27900000000000003,"15min":0.27100000000000002},"min":{"1min":0.20999999999999999,"5min":0.185,"15min":0.17499999999999999},"max":{"1min":0.44400000000000001,"5min":0.47199999999999998,"15min":0.47199999999999998},"last":0.27900000000000003}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051803,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:44:55 2023","interfaces":[{"interface":"back","average":{"1min":0.252,"5min":0.26500000000000001,"15min":0.26900000000000002},"min":{"1min":0.183,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.39700000000000002,"5min":0.44,"15min":0.44},"last":0.20699999999999999},{"interface":"front","average":{"1min":0.27000000000000002,"5min":0.28100000000000003,"15min":0.28299999999999997},"min":{"1min":0.20300000000000001,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":0.42099999999999999,"5min":0.47599999999999998,"15min":0.48599999999999999},"last":0.23599999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:45:25.659 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:45:25.969 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:45:42.735+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:45:42.744+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:45:42.897 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace: 2023-03-25T04:46:00.183 INFO:teuthology.orchestra.run.smithi070.stdout:
, 2023-03-25T04:46:00.183 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":1217,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":20,"modified":"2023-03-25T04:45:42.307700+0000","services":{"mgr":{"daemons":{"summary":"","x":{"start_epoch":0,"start_stamp":"0.000000","gid":0,"addr":"(unrecognized address family 0)/0","metadata":{},"task_status":{}}}}}},"progress_events":{}}
, 2023-03-25T04:46:00.197 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now

Ignoring malparsed backtrace: 2023-03-25T04:46:19.910+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:46:20.039 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all

Ignoring malparsed backtrace: 2023-03-25T04:46:39.611 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}
, 2023-03-25T04:46:39.611 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json
, 2023-03-25T04:46:39.613+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:46:58.485+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:46:58.764+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:46:59.519+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:47:14.132 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:47:14.146 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":678,"stamp":"2023-03-25T04:47:12.325044+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002571"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":42,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:44.393426+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:44:44.393426+0000","last_peered":"2023-03-25T04:44:44.393426+0000","last_clean":"2023-03-25T04:44:44.393426+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:44:44.393426+0000","last_undegraded":"2023-03-25T04:44:44.393426+0000","last_fullsized":"2023-03-25T04:44:44.393426+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:39.393044+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:44:39.393044+0000","last_peered":"2023-03-25T04:44:39.393044+0000","last_clean":"2023-03-25T04:44:39.393044+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:44:39.393044+0000","last_undegraded":"2023-03-25T04:44:39.393044+0000","last_fullsized":"2023-03-25T04:44:39.393044+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:43:24.386177+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:43:24.386177+0000","last_peered":"2023-03-25T04:43:24.386177+0000","last_clean":"2023-03-25T04:43:24.386177+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:43:24.386177+0000","last_undegraded":"2023-03-25T04:43:24.386177+0000","last_fullsized":"2023-03-25T04:43:24.386177+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:43:55.629798+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:43:55.629798+0000","last_peered":"2023-03-25T04:43:55.629798+0000","last_clean":"2023-03-25T04:43:55.629798+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:43:55.629798+0000","last_undegraded":"2023-03-25T04:43:55.629798+0000","last_fullsized":"2023-03-25T04:43:55.629798+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":42,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:04.389904+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:44:04.389904+0000","last_peered":"2023-03-25T04:44:04.389904+0000","last_clean":"2023-03-25T04:44:04.389904+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:44:04.389904+0000","last_undegraded":"2023-03-25T04:44:04.389904+0000","last_fullsized":"2023-03-25T04:44:04.389904+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:04.389923+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:44:04.389923+0000","last_peered":"2023-03-25T04:44:04.389923+0000","last_clean":"2023-03-25T04:44:04.389923+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:44:04.389923+0000","last_undegraded":"2023-03-25T04:44:04.389923+0000","last_fullsized":"2023-03-25T04:44:04.389923+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117235,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:47:05 2023","interfaces":[{"interface":"back","average":{"1min":0.27800000000000002,"5min":0.27000000000000002,"15min":0.26300000000000001},"min":{"1min":0.20100000000000001,"5min":0.16900000000000001,"15min":0.16},"max":{"1min":0.39200000000000002,"5min":0.42999999999999999,"15min":0.45600000000000002},"last":0.379},{"interface":"front","average":{"1min":0.28399999999999997,"5min":0.28100000000000003,"15min":0.27200000000000002},"min":{"1min":0.17799999999999999,"5min":0.17799999999999999,"15min":0.17699999999999999},"max":{"1min":0.42599999999999999,"5min":0.47199999999999998,"15min":0.47199999999999998},"last":0.39800000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051825,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:47:01 2023","interfaces":[{"interface":"back","average":{"1min":0.27300000000000002,"5min":0.26100000000000001,"15min":0.27000000000000002},"min":{"1min":0.186,"5min":0.17399999999999999,"15min":0.17000000000000001},"max":{"1min":0.40400000000000003,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.23899999999999999},{"interface":"front","average":{"1min":0.28699999999999998,"5min":0.27500000000000002,"15min":0.28299999999999997},"min":{"1min":0.186,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.48599999999999999},"last":0.19600000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:47:31.198 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json
, 2023-03-25T04:47:31.411+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:47:31.789 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:47:49.509 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":696,"stamp":"2023-03-25T04:47:48.332093+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002697"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":42,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:44.393426+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:44:44.393426+0000","last_peered":"2023-03-25T04:44:44.393426+0000","last_clean":"2023-03-25T04:44:44.393426+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:44:44.393426+0000","last_undegraded":"2023-03-25T04:44:44.393426+0000","last_fullsized":"2023-03-25T04:44:44.393426+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:39.393044+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:44:39.393044+0000","last_peered":"2023-03-25T04:44:39.393044+0000","last_clean":"2023-03-25T04:44:39.393044+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:44:39.393044+0000","last_undegraded":"2023-03-25T04:44:39.393044+0000","last_fullsized":"2023-03-25T04:44:39.393044+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:45:24.396717+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:45:24.396717+0000","last_peered":"2023-03-25T04:45:24.396717+0000","last_clean":"2023-03-25T04:45:24.396717+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:45:24.396717+0000","last_undegraded":"2023-03-25T04:45:24.396717+0000","last_fullsized":"2023-03-25T04:45:24.396717+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:43:55.629798+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:43:55.629798+0000","last_peered":"2023-03-25T04:43:55.629798+0000","last_clean":"2023-03-25T04:43:55.629798+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:43:55.629798+0000","last_undegraded":"2023-03-25T04:43:55.629798+0000","last_fullsized":"2023-03-25T04:43:55.629798+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":42,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:04.389904+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:44:04.389904+0000","last_peered":"2023-03-25T04:44:04.389904+0000","last_clean":"2023-03-25T04:44:04.389904+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:44:04.389904+0000","last_undegraded":"2023-03-25T04:44:04.389904+0000","last_fullsized":"2023-03-25T04:44:04.389904+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:04.389923+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:44:04.389923+0000","last_peered":"2023-03-25T04:44:04.389923+0000","last_clean":"2023-03-25T04:44:04.389923+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:44:04.389923+0000","last_undegraded":"2023-03-25T04:44:04.389923+0000","last_fullsized":"2023-03-25T04:44:04.389923+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117242,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:47:05 2023","interfaces":[{"interface":"back","average":{"1min":0.27800000000000002,"5min":0.27000000000000002,"15min":0.26300000000000001},"min":{"1min":0.20100000000000001,"5min":0.16900000000000001,"15min":0.16},"max":{"1min":0.39200000000000002,"5min":0.42999999999999999,"15min":0.45600000000000002},"last":0.40500000000000003},{"interface":"front","average":{"1min":0.28399999999999997,"5min":0.28100000000000003,"15min":0.27200000000000002},"min":{"1min":0.17799999999999999,"5min":0.17799999999999999,"15min":0.17699999999999999},"max":{"1min":0.42599999999999999,"5min":0.47199999999999998,"15min":0.47199999999999998},"last":0.42399999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051834,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:47:01 2023","interfaces":[{"interface":"back","average":{"1min":0.27300000000000002,"5min":0.26100000000000001,"15min":0.27000000000000002},"min":{"1min":0.186,"5min":0.17399999999999999,"15min":0.17000000000000001},"max":{"1min":0.40400000000000003,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.25600000000000001},{"interface":"front","average":{"1min":0.28699999999999998,"5min":0.27500000000000002,"15min":0.28299999999999997},"min":{"1min":0.186,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.48599999999999999},"last":0.22800000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:47:49.510 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace:     ctx.managers[config['cluster']].wait_for_clean()
,   File "/home/teuthworker/src/github.com_ceph_ceph-c_511015c23eef0684a64171b710e432448df2b7c7/qa/tasks/ceph_manager.py", line 2747, in wait_for_clean
,     assert time.time() - start < timeout, \

Ignoring malparsed backtrace:  ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
,  1: /lib64/libpthread.so.0(+0x12cf0) [0x7f1453c2fcf0]
,  2: __pthread_mutex_lock()

Ignoring malparsed backtrace: 2023-03-25T04:28:05.172 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":142,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":5,"modified":"2023-03-25T04:27:12.092098+0000","services":{}},"progress_events":{}}
, 2023-03-25T04:28:05.185 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now
, 2023-03-25T04:28:05.186 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace: 2023-03-25T04:28:28.360 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":116,"stamp":"2023-03-25T04:28:28.108007+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002578"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.314661+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:27:45.314661+0000","last_peered":"2023-03-25T04:27:45.314661+0000","last_clean":"2023-03-25T04:27:45.314661+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:27:45.314661+0000","last_undegraded":"2023-03-25T04:27:45.314661+0000","last_fullsized":"2023-03-25T04:27:45.314661+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":25,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.310745+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:27:44.310745+0000","last_peered":"2023-03-25T04:27:44.310745+0000","last_clean":"2023-03-25T04:27:44.310745+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:27:44.310745+0000","last_undegraded":"2023-03-25T04:27:44.310745+0000","last_fullsized":"2023-03-25T04:27:44.310745+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":25,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.310650+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:27:44.310650+0000","last_peered":"2023-03-25T04:27:44.310650+0000","last_clean":"2023-03-25T04:27:44.310650+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:27:44.310650+0000","last_undegraded":"2023-03-25T04:27:44.310650+0000","last_fullsized":"2023-03-25T04:27:44.310650+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":26,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.312962+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:27:44.312962+0000","last_peered":"2023-03-25T04:27:44.312962+0000","last_clean":"2023-03-25T04:27:44.312962+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:27:44.312962+0000","last_undegraded":"2023-03-25T04:27:44.312962+0000","last_fullsized":"2023-03-25T04:27:44.312962+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315064+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:27:45.315064+0000","last_peered":"2023-03-25T04:27:45.315064+0000","last_clean":"2023-03-25T04:27:45.315064+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:27:45.315064+0000","last_undegraded":"2023-03-25T04:27:45.315064+0000","last_fullsized":"2023-03-25T04:27:45.315064+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":25,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.310707+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:27:44.310707+0000","last_peered":"2023-03-25T04:27:44.310707+0000","last_clean":"2023-03-25T04:27:44.310707+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:27:44.310707+0000","last_undegraded":"2023-03-25T04:27:44.310707+0000","last_fullsized":"2023-03-25T04:27:44.310707+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117007,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.375}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051597,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.186}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:28:28.362 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean
, 2023-03-25T04:28:28.362 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:28:45.655 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":124,"stamp":"2023-03-25T04:28:44.111265+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002492"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.314661+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:27:45.314661+0000","last_peered":"2023-03-25T04:27:45.314661+0000","last_clean":"2023-03-25T04:27:45.314661+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:27:45.314661+0000","last_undegraded":"2023-03-25T04:27:45.314661+0000","last_fullsized":"2023-03-25T04:27:45.314661+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":25,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.310745+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:27:44.310745+0000","last_peered":"2023-03-25T04:27:44.310745+0000","last_clean":"2023-03-25T04:27:44.310745+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:27:44.310745+0000","last_undegraded":"2023-03-25T04:27:44.310745+0000","last_fullsized":"2023-03-25T04:27:44.310745+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":25,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.310650+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:27:44.310650+0000","last_peered":"2023-03-25T04:27:44.310650+0000","last_clean":"2023-03-25T04:27:44.310650+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:27:44.310650+0000","last_undegraded":"2023-03-25T04:27:44.310650+0000","last_fullsized":"2023-03-25T04:27:44.310650+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":26,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.312962+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:27:44.312962+0000","last_peered":"2023-03-25T04:27:44.312962+0000","last_clean":"2023-03-25T04:27:44.312962+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:27:44.312962+0000","last_undegraded":"2023-03-25T04:27:44.312962+0000","last_fullsized":"2023-03-25T04:27:44.312962+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315064+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:27:45.315064+0000","last_peered":"2023-03-25T04:27:45.315064+0000","last_clean":"2023-03-25T04:27:45.315064+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:27:45.315064+0000","last_undegraded":"2023-03-25T04:27:45.315064+0000","last_fullsized":"2023-03-25T04:27:45.315064+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":25,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.310707+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:27:44.310707+0000","last_peered":"2023-03-25T04:27:44.310707+0000","last_clean":"2023-03-25T04:27:44.310707+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:27:44.310707+0000","last_undegraded":"2023-03-25T04:27:44.310707+0000","last_fullsized":"2023-03-25T04:27:44.310707+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117011,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:28:31 2023","interfaces":[{"interface":"back","average":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.36899999999999999},{"interface":"front","average":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"min":{"1min":0.19,"5min":0.19,"15min":0.19},"max":{"1min":0.46300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.39800000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051601,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:28:24 2023","interfaces":[{"interface":"back","average":{"1min":0.249,"5min":0.249,"15min":0.249},"min":{"1min":0.17699999999999999,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"last":0.254},{"interface":"front","average":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"min":{"1min":0.20200000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"last":0.312}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:28:45.656 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace: 2023-03-25T04:29:05.171 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}
, 2023-03-25T04:29:05.171 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json
, 2023-03-25T04:29:05.561 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:29:22.465 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:29:22.773 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all

Ignoring malparsed backtrace: 2023-03-25T04:29:42.405 INFO:teuthology.orchestra.run.smithi070.stdout:
, 2023-03-25T04:29:42.406 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":240,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":9,"modified":"2023-03-25T04:29:32.120358+0000","services":{"mon":{"daemons":{"summary":"","a":{"start_epoch":0,"start_stamp":"0.000000","gid":0,"addr":"(unrecognized address family 0)/0","metadata":{},"task_status":{}}}}}},"progress_events":{}}
, 2023-03-25T04:29:42.419 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:29:59.627 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":161,"stamp":"2023-03-25T04:29:58.125186+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002398"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315468+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:27:45.315468+0000","last_peered":"2023-03-25T04:27:45.315468+0000","last_clean":"2023-03-25T04:27:45.315468+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:27:45.315468+0000","last_undegraded":"2023-03-25T04:27:45.315468+0000","last_fullsized":"2023-03-25T04:27:45.315468+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312845+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:27:45.312845+0000","last_peered":"2023-03-25T04:27:45.312845+0000","last_clean":"2023-03-25T04:27:45.312845+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:27:45.312845+0000","last_undegraded":"2023-03-25T04:27:45.312845+0000","last_fullsized":"2023-03-25T04:27:45.312845+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312882+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:27:45.312882+0000","last_peered":"2023-03-25T04:27:45.312882+0000","last_clean":"2023-03-25T04:27:45.312882+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:27:45.312882+0000","last_undegraded":"2023-03-25T04:27:45.312882+0000","last_fullsized":"2023-03-25T04:27:45.312882+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.311730+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:27:45.311730+0000","last_peered":"2023-03-25T04:27:45.311730+0000","last_clean":"2023-03-25T04:27:45.311730+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:27:45.311730+0000","last_undegraded":"2023-03-25T04:27:45.311730+0000","last_fullsized":"2023-03-25T04:27:45.311730+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315735+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:27:45.315735+0000","last_peered":"2023-03-25T04:27:45.315735+0000","last_clean":"2023-03-25T04:27:45.315735+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:27:45.315735+0000","last_undegraded":"2023-03-25T04:27:45.315735+0000","last_fullsized":"2023-03-25T04:27:45.315735+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312833+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:27:45.312833+0000","last_peered":"2023-03-25T04:27:45.312833+0000","last_clean":"2023-03-25T04:27:45.312833+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:27:45.312833+0000","last_undegraded":"2023-03-25T04:27:45.312833+0000","last_fullsized":"2023-03-25T04:27:45.312833+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117026,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:29:33 2023","interfaces":[{"interface":"back","average":{"1min":0.26300000000000001,"5min":0.29099999999999998,"15min":0.29599999999999999},"min":{"1min":0.191,"5min":0.191,"15min":0.191},"max":{"1min":0.371,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.20499999999999999},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.29599999999999999,"15min":0.29999999999999999},"min":{"1min":0.21099999999999999,"5min":0.19,"15min":0.19},"max":{"1min":0.39800000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.19500000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051616,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:29:29 2023","interfaces":[{"interface":"back","average":{"1min":0.27200000000000002,"5min":0.253,"15min":0.25},"min":{"1min":0.183,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"last":0.26400000000000001},{"interface":"front","average":{"1min":0.28499999999999998,"5min":0.26800000000000002,"15min":0.26500000000000001},"min":{"1min":0.16600000000000001,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"last":0.29899999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:29:59.629 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean

Ignoring malparsed backtrace: 2023-03-25T04:30:16.889 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:30:16.903 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":170,"stamp":"2023-03-25T04:30:16.128427+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002222"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315468+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:27:45.315468+0000","last_peered":"2023-03-25T04:27:45.315468+0000","last_clean":"2023-03-25T04:27:45.315468+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:27:45.315468+0000","last_undegraded":"2023-03-25T04:27:45.315468+0000","last_fullsized":"2023-03-25T04:27:45.315468+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312845+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:27:45.312845+0000","last_peered":"2023-03-25T04:27:45.312845+0000","last_clean":"2023-03-25T04:27:45.312845+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:27:45.312845+0000","last_undegraded":"2023-03-25T04:27:45.312845+0000","last_fullsized":"2023-03-25T04:27:45.312845+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312882+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:27:45.312882+0000","last_peered":"2023-03-25T04:27:45.312882+0000","last_clean":"2023-03-25T04:27:45.312882+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:27:45.312882+0000","last_undegraded":"2023-03-25T04:27:45.312882+0000","last_fullsized":"2023-03-25T04:27:45.312882+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.311730+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:27:45.311730+0000","last_peered":"2023-03-25T04:27:45.311730+0000","last_clean":"2023-03-25T04:27:45.311730+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:27:45.311730+0000","last_undegraded":"2023-03-25T04:27:45.311730+0000","last_fullsized":"2023-03-25T04:27:45.311730+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315735+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:27:45.315735+0000","last_peered":"2023-03-25T04:27:45.315735+0000","last_clean":"2023-03-25T04:27:45.315735+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:27:45.315735+0000","last_undegraded":"2023-03-25T04:27:45.315735+0000","last_fullsized":"2023-03-25T04:27:45.315735+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312833+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:27:45.312833+0000","last_peered":"2023-03-25T04:27:45.312833+0000","last_clean":"2023-03-25T04:27:45.312833+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:27:45.312833+0000","last_undegraded":"2023-03-25T04:27:45.312833+0000","last_fullsized":"2023-03-25T04:27:45.312833+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117030,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:29:33 2023","interfaces":[{"interface":"back","average":{"1min":0.26300000000000001,"5min":0.29099999999999998,"15min":0.29599999999999999},"min":{"1min":0.191,"5min":0.191,"15min":0.191},"max":{"1min":0.371,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.245},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.29599999999999999,"15min":0.29999999999999999},"min":{"1min":0.21099999999999999,"5min":0.19,"15min":0.19},"max":{"1min":0.39800000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.20899999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051620,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:29:29 2023","interfaces":[{"interface":"back","average":{"1min":0.27200000000000002,"5min":0.253,"15min":0.25},"min":{"1min":0.183,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"last":0.19400000000000001},{"interface":"front","average":{"1min":0.28499999999999998,"5min":0.26800000000000002,"15min":0.26500000000000001},"min":{"1min":0.16600000000000001,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"last":0.23599999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:30:31.984+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:30:32.201+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:30:32.981+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)

Ignoring malparsed backtrace: 2023-03-25T04:30:48.449 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now
, 2023-03-25T04:30:48.450 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:30:48.760 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:31:08.342 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":196,"stamp":"2023-03-25T04:31:08.139018+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002431"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315468+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:27:45.315468+0000","last_peered":"2023-03-25T04:27:45.315468+0000","last_clean":"2023-03-25T04:27:45.315468+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:27:45.315468+0000","last_undegraded":"2023-03-25T04:27:45.315468+0000","last_fullsized":"2023-03-25T04:27:45.315468+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312845+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:27:45.312845+0000","last_peered":"2023-03-25T04:27:45.312845+0000","last_clean":"2023-03-25T04:27:45.312845+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:27:45.312845+0000","last_undegraded":"2023-03-25T04:27:45.312845+0000","last_fullsized":"2023-03-25T04:27:45.312845+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312882+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:27:45.312882+0000","last_peered":"2023-03-25T04:27:45.312882+0000","last_clean":"2023-03-25T04:27:45.312882+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:27:45.312882+0000","last_undegraded":"2023-03-25T04:27:45.312882+0000","last_fullsized":"2023-03-25T04:27:45.312882+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.311730+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:27:45.311730+0000","last_peered":"2023-03-25T04:27:45.311730+0000","last_clean":"2023-03-25T04:27:45.311730+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:27:45.311730+0000","last_undegraded":"2023-03-25T04:27:45.311730+0000","last_fullsized":"2023-03-25T04:27:45.311730+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315735+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:27:45.315735+0000","last_peered":"2023-03-25T04:27:45.315735+0000","last_clean":"2023-03-25T04:27:45.315735+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:27:45.315735+0000","last_undegraded":"2023-03-25T04:27:45.315735+0000","last_fullsized":"2023-03-25T04:27:45.315735+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312833+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:27:45.312833+0000","last_peered":"2023-03-25T04:27:45.312833+0000","last_clean":"2023-03-25T04:27:45.312833+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:27:45.312833+0000","last_undegraded":"2023-03-25T04:27:45.312833+0000","last_fullsized":"2023-03-25T04:27:45.312833+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117040,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:30:37 2023","interfaces":[{"interface":"back","average":{"1min":0.28799999999999998,"5min":0.28899999999999998,"15min":0.29499999999999998},"min":{"1min":0.20200000000000001,"5min":0.191,"15min":0.191},"max":{"1min":0.433,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.29799999999999999},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.28999999999999998,"15min":0.29799999999999999},"min":{"1min":0.17499999999999999,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":0.40300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.25700000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051630,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:30:32 2023","interfaces":[{"interface":"back","average":{"1min":0.25600000000000001,"5min":0.255,"15min":0.251},"min":{"1min":0.19400000000000001,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.35699999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"last":0.22800000000000001},{"interface":"front","average":{"1min":0.28699999999999998,"5min":0.27200000000000002,"15min":0.26600000000000001},"min":{"1min":0.20599999999999999,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.39400000000000002,"5min":0.42899999999999999,"15min":0.42899999999999999},"last":0.24099999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:31:08.343 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace: 2023-03-25T04:31:24.283+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:31:25.192+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:31:25.295+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)

Ignoring malparsed backtrace: 2023-03-25T04:31:40.257 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:31:40.271 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":212,"stamp":"2023-03-25T04:31:40.144732+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002208"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315468+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:27:45.315468+0000","last_peered":"2023-03-25T04:27:45.315468+0000","last_clean":"2023-03-25T04:27:45.315468+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:27:45.315468+0000","last_undegraded":"2023-03-25T04:27:45.315468+0000","last_fullsized":"2023-03-25T04:27:45.315468+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312845+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:27:45.312845+0000","last_peered":"2023-03-25T04:27:45.312845+0000","last_clean":"2023-03-25T04:27:45.312845+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:27:45.312845+0000","last_undegraded":"2023-03-25T04:27:45.312845+0000","last_fullsized":"2023-03-25T04:27:45.312845+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:23.308607+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:29:23.308607+0000","last_peered":"2023-03-25T04:29:23.308607+0000","last_clean":"2023-03-25T04:29:23.308607+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:29:23.308607+0000","last_undegraded":"2023-03-25T04:29:23.308607+0000","last_fullsized":"2023-03-25T04:29:23.308607+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":28,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:20.554710+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:29:20.554710+0000","last_peered":"2023-03-25T04:29:20.554710+0000","last_clean":"2023-03-25T04:29:20.554710+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:29:20.554710+0000","last_undegraded":"2023-03-25T04:29:20.554710+0000","last_fullsized":"2023-03-25T04:29:20.554710+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309216+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:29:28.309216+0000","last_peered":"2023-03-25T04:29:28.309216+0000","last_clean":"2023-03-25T04:29:28.309216+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:29:28.309216+0000","last_undegraded":"2023-03-25T04:29:28.309216+0000","last_fullsized":"2023-03-25T04:29:28.309216+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309233+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:29:28.309233+0000","last_peered":"2023-03-25T04:29:28.309233+0000","last_clean":"2023-03-25T04:29:28.309233+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:29:28.309233+0000","last_undegraded":"2023-03-25T04:29:28.309233+0000","last_fullsized":"2023-03-25T04:29:28.309233+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117046,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:30:37 2023","interfaces":[{"interface":"back","average":{"1min":0.28799999999999998,"5min":0.28899999999999998,"15min":0.29499999999999998},"min":{"1min":0.20200000000000001,"5min":0.191,"15min":0.191},"max":{"1min":0.433,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.216},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.28999999999999998,"15min":0.29799999999999999},"min":{"1min":0.17499999999999999,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":0.40300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.24299999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051637,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:31:35 2023","interfaces":[{"interface":"back","average":{"1min":0.26100000000000001,"5min":0.25700000000000001,"15min":0.251},"min":{"1min":0.19,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.35899999999999999,"5min":0.41599999999999998,"15min":0.41599999999999998},"last":0.221},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.27400000000000002,"15min":0.26700000000000002},"min":{"1min":0.20200000000000001,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.374,"5min":0.42899999999999999,"15min":0.42899999999999999},"last":0.24099999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:32:01.906+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:32:02.388+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:32:02.794 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:32:22.326 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":233,"stamp":"2023-03-25T04:32:22.153289+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002545"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:33.309748+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:29:33.309748+0000","last_peered":"2023-03-25T04:29:33.309748+0000","last_clean":"2023-03-25T04:29:33.309748+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:29:33.309748+0000","last_undegraded":"2023-03-25T04:29:33.309748+0000","last_fullsized":"2023-03-25T04:29:33.309748+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:33.309729+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:29:33.309729+0000","last_peered":"2023-03-25T04:29:33.309729+0000","last_clean":"2023-03-25T04:29:33.309729+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:29:33.309729+0000","last_undegraded":"2023-03-25T04:29:33.309729+0000","last_fullsized":"2023-03-25T04:29:33.309729+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:23.308607+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:29:23.308607+0000","last_peered":"2023-03-25T04:29:23.308607+0000","last_clean":"2023-03-25T04:29:23.308607+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:29:23.308607+0000","last_undegraded":"2023-03-25T04:29:23.308607+0000","last_fullsized":"2023-03-25T04:29:23.308607+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":28,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:20.554710+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:29:20.554710+0000","last_peered":"2023-03-25T04:29:20.554710+0000","last_clean":"2023-03-25T04:29:20.554710+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:29:20.554710+0000","last_undegraded":"2023-03-25T04:29:20.554710+0000","last_fullsized":"2023-03-25T04:29:20.554710+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309216+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:29:28.309216+0000","last_peered":"2023-03-25T04:29:28.309216+0000","last_clean":"2023-03-25T04:29:28.309216+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:29:28.309216+0000","last_undegraded":"2023-03-25T04:29:28.309216+0000","last_fullsized":"2023-03-25T04:29:28.309216+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309233+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:29:28.309233+0000","last_peered":"2023-03-25T04:29:28.309233+0000","last_clean":"2023-03-25T04:29:28.309233+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:29:28.309233+0000","last_undegraded":"2023-03-25T04:29:28.309233+0000","last_fullsized":"2023-03-25T04:29:28.309233+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117055,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:31:42 2023","interfaces":[{"interface":"back","average":{"1min":0.27500000000000002,"5min":0.28399999999999997,"15min":0.29399999999999998},"min":{"1min":0.19800000000000001,"5min":0.191,"15min":0.191},"max":{"1min":0.36899999999999999,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.32300000000000001},{"interface":"front","average":{"1min":0.27900000000000003,"5min":0.28499999999999998,"15min":0.29599999999999999},"min":{"1min":0.216,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":0.40799999999999997,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.27100000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051646,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:31:35 2023","interfaces":[{"interface":"back","average":{"1min":0.26100000000000001,"5min":0.25700000000000001,"15min":0.251},"min":{"1min":0.19,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.35899999999999999,"5min":0.41599999999999998,"15min":0.41599999999999998},"last":0.25600000000000001},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.27400000000000002,"15min":0.26700000000000002},"min":{"1min":0.20200000000000001,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.374,"5min":0.42899999999999999,"15min":0.42899999999999999},"last":0.29199999999999998}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:32:22.327 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean
, 2023-03-25T04:32:22.327 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}

Ignoring malparsed backtrace: 2023-03-25T04:32:39.836 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:32:39.849 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":241,"stamp":"2023-03-25T04:32:38.156312+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002346"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:33.309748+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:29:33.309748+0000","last_peered":"2023-03-25T04:29:33.309748+0000","last_clean":"2023-03-25T04:29:33.309748+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:29:33.309748+0000","last_undegraded":"2023-03-25T04:29:33.309748+0000","last_fullsized":"2023-03-25T04:29:33.309748+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:33.309729+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:29:33.309729+0000","last_peered":"2023-03-25T04:29:33.309729+0000","last_clean":"2023-03-25T04:29:33.309729+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:29:33.309729+0000","last_undegraded":"2023-03-25T04:29:33.309729+0000","last_fullsized":"2023-03-25T04:29:33.309729+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:23.308607+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:29:23.308607+0000","last_peered":"2023-03-25T04:29:23.308607+0000","last_clean":"2023-03-25T04:29:23.308607+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:29:23.308607+0000","last_undegraded":"2023-03-25T04:29:23.308607+0000","last_fullsized":"2023-03-25T04:29:23.308607+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":28,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:20.554710+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:29:20.554710+0000","last_peered":"2023-03-25T04:29:20.554710+0000","last_clean":"2023-03-25T04:29:20.554710+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:29:20.554710+0000","last_undegraded":"2023-03-25T04:29:20.554710+0000","last_fullsized":"2023-03-25T04:29:20.554710+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309216+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:29:28.309216+0000","last_peered":"2023-03-25T04:29:28.309216+0000","last_clean":"2023-03-25T04:29:28.309216+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:29:28.309216+0000","last_undegraded":"2023-03-25T04:29:28.309216+0000","last_fullsized":"2023-03-25T04:29:28.309216+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309233+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:29:28.309233+0000","last_peered":"2023-03-25T04:29:28.309233+0000","last_clean":"2023-03-25T04:29:28.309233+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:29:28.309233+0000","last_undegraded":"2023-03-25T04:29:28.309233+0000","last_fullsized":"2023-03-25T04:29:28.309233+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117058,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:31:42 2023","interfaces":[{"interface":"back","average":{"1min":0.27500000000000002,"5min":0.28399999999999997,"15min":0.29399999999999998},"min":{"1min":0.19800000000000001,"5min":0.191,"15min":0.191},"max":{"1min":0.36899999999999999,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.30599999999999999},{"interface":"front","average":{"1min":0.27900000000000003,"5min":0.28499999999999998,"15min":0.29599999999999999},"min":{"1min":0.216,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":0.40799999999999997,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.32500000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051649,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:31:35 2023","interfaces":[{"interface":"back","average":{"1min":0.26100000000000001,"5min":0.25700000000000001,"15min":0.251},"min":{"1min":0.19,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.35899999999999999,"5min":0.41599999999999998,"15min":0.41599999999999998},"last":0.24199999999999999},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.27400000000000002,"15min":0.26700000000000002},"min":{"1min":0.20200000000000001,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.374,"5min":0.42899999999999999,"15min":0.42899999999999999},"last":0.22600000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:32:54.944+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:32:55.469+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:32:55.927+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)

Ignoring malparsed backtrace: 2023-03-25T04:33:13.958 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean
, 2023-03-25T04:33:13.959 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}
, 2023-03-25T04:33:13.959 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json

Ignoring malparsed backtrace: 2023-03-25T04:33:31.199 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":267,"stamp":"2023-03-25T04:33:30.166660+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002269"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:33.309748+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:29:33.309748+0000","last_peered":"2023-03-25T04:29:33.309748+0000","last_clean":"2023-03-25T04:29:33.309748+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:29:33.309748+0000","last_undegraded":"2023-03-25T04:29:33.309748+0000","last_fullsized":"2023-03-25T04:29:33.309748+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:33.309729+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:29:33.309729+0000","last_peered":"2023-03-25T04:29:33.309729+0000","last_clean":"2023-03-25T04:29:33.309729+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:29:33.309729+0000","last_undegraded":"2023-03-25T04:29:33.309729+0000","last_fullsized":"2023-03-25T04:29:33.309729+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":28,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:31:23.320894+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:31:23.320894+0000","last_peered":"2023-03-25T04:31:23.320894+0000","last_clean":"2023-03-25T04:31:23.320894+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:31:23.320894+0000","last_undegraded":"2023-03-25T04:31:23.320894+0000","last_fullsized":"2023-03-25T04:31:23.320894+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":28,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:20.554710+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:29:20.554710+0000","last_peered":"2023-03-25T04:29:20.554710+0000","last_clean":"2023-03-25T04:29:20.554710+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:29:20.554710+0000","last_undegraded":"2023-03-25T04:29:20.554710+0000","last_fullsized":"2023-03-25T04:29:20.554710+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309216+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:29:28.309216+0000","last_peered":"2023-03-25T04:29:28.309216+0000","last_clean":"2023-03-25T04:29:28.309216+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:29:28.309216+0000","last_undegraded":"2023-03-25T04:29:28.309216+0000","last_fullsized":"2023-03-25T04:29:28.309216+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309233+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:29:28.309233+0000","last_peered":"2023-03-25T04:29:28.309233+0000","last_clean":"2023-03-25T04:29:28.309233+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:29:28.309233+0000","last_undegraded":"2023-03-25T04:29:28.309233+0000","last_fullsized":"2023-03-25T04:29:28.309233+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117068,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:32:46 2023","interfaces":[{"interface":"back","average":{"1min":0.27000000000000002,"5min":0.27900000000000003,"15min":0.29199999999999998},"min":{"1min":0.17899999999999999,"5min":0.17899999999999999,"15min":0.17899999999999999},"max":{"1min":0.35599999999999998,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.28799999999999998},{"interface":"front","average":{"1min":0.26800000000000002,"5min":0.27800000000000002,"15min":0.29399999999999998},"min":{"1min":0.19900000000000001,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":0.377,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.315}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051660,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:32:36 2023","interfaces":[{"interface":"back","average":{"1min":0.248,"5min":0.25700000000000001,"15min":0.251},"min":{"1min":0.17000000000000001,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"last":0.22700000000000001},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.27600000000000002,"15min":0.26800000000000002},"min":{"1min":0.192,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"last":0.20399999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:33:31.201 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:33:31.400+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)

Ignoring malparsed backtrace: 2023-03-25T04:33:47.531+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:33:47.951+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:33:48.505+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)

Ignoring malparsed backtrace: 2023-03-25T04:34:06.104 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json
, 2023-03-25T04:34:06.118+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:34:06.497 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:34:23.399 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:34:23.665+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:34:23.712 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:34:40.195+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:34:40.523+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:34:40.676 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace: 2023-03-25T04:35:03.381+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:35:04.139+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:35:04.391+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)

Ignoring malparsed backtrace: 2023-03-25T04:35:22.738 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean
, 2023-03-25T04:35:22.739 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}
, 2023-03-25T04:35:22.739 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json

Ignoring malparsed backtrace: 2023-03-25T04:35:41.393+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:35:42.352+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:35:42.376+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:35:57.384 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":340,"stamp":"2023-03-25T04:35:56.194992+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.001930"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":36,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:31:43.322894+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:31:43.322894+0000","last_peered":"2023-03-25T04:31:43.322894+0000","last_clean":"2023-03-25T04:31:43.322894+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:31:43.322894+0000","last_undegraded":"2023-03-25T04:31:43.322894+0000","last_fullsized":"2023-03-25T04:31:43.322894+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":28,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:31:43.322876+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:31:43.322876+0000","last_peered":"2023-03-25T04:31:43.322876+0000","last_clean":"2023-03-25T04:31:43.322876+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:31:43.322876+0000","last_undegraded":"2023-03-25T04:31:43.322876+0000","last_fullsized":"2023-03-25T04:31:43.322876+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:23.331998+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:33:23.331998+0000","last_peered":"2023-03-25T04:33:23.331998+0000","last_clean":"2023-03-25T04:33:23.331998+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:33:23.331998+0000","last_undegraded":"2023-03-25T04:33:23.331998+0000","last_fullsized":"2023-03-25T04:33:23.331998+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:30.576723+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:33:30.576723+0000","last_peered":"2023-03-25T04:33:30.576723+0000","last_clean":"2023-03-25T04:33:30.576723+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:33:30.576723+0000","last_undegraded":"2023-03-25T04:33:30.576723+0000","last_fullsized":"2023-03-25T04:33:30.576723+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":37,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:38.333447+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:33:38.333447+0000","last_peered":"2023-03-25T04:33:38.333447+0000","last_clean":"2023-03-25T04:33:38.333447+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:33:38.333447+0000","last_undegraded":"2023-03-25T04:33:38.333447+0000","last_fullsized":"2023-03-25T04:33:38.333447+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:38.333467+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:33:38.333467+0000","last_peered":"2023-03-25T04:33:38.333467+0000","last_clean":"2023-03-25T04:33:38.333467+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:33:38.333467+0000","last_undegraded":"2023-03-25T04:33:38.333467+0000","last_fullsized":"2023-03-25T04:33:38.333467+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117098,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:35:49 2023","interfaces":[{"interface":"back","average":{"1min":0.255,"5min":0.26000000000000001,"15min":0.28199999999999997},"min":{"1min":0.17899999999999999,"5min":0.16,"15min":0.16},"max":{"1min":0.36399999999999999,"5min":0.42599999999999999,"15min":0.45100000000000001},"last":0.26100000000000001},{"interface":"front","average":{"1min":0.26000000000000001,"5min":0.26700000000000002,"15min":0.28599999999999998},"min":{"1min":0.188,"5min":0.17999999999999999,"15min":0.17499999999999999},"max":{"1min":0.39100000000000001,"5min":0.45200000000000001,"15min":0.46300000000000002},"last":0.28399999999999997}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051689,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:35:42 2023","interfaces":[{"interface":"back","average":{"1min":0.29399999999999998,"5min":0.27200000000000002,"15min":0.25800000000000001},"min":{"1min":0.17299999999999999,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.41099999999999998,"5min":0.42699999999999999,"15min":0.42699999999999999},"last":0.20399999999999999},{"interface":"front","average":{"1min":0.30299999999999999,"5min":0.28399999999999997,"15min":0.27300000000000002},"min":{"1min":0.19700000000000001,"5min":0.192,"15min":0.16600000000000001},"max":{"1min":0.43099999999999999,"5min":0.45800000000000002,"15min":0.45800000000000002},"last":0.22}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:35:57.624+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)

Ignoring malparsed backtrace: 2023-03-25T04:36:17.226 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:36:17.239 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":350,"stamp":"2023-03-25T04:36:16.198866+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002148"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":37,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:53.335026+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:33:53.335026+0000","last_peered":"2023-03-25T04:33:53.335026+0000","last_clean":"2023-03-25T04:33:53.335026+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:33:53.335026+0000","last_undegraded":"2023-03-25T04:33:53.335026+0000","last_fullsized":"2023-03-25T04:33:53.335026+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:53.335009+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:33:53.335009+0000","last_peered":"2023-03-25T04:33:53.335009+0000","last_clean":"2023-03-25T04:33:53.335009+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:33:53.335009+0000","last_undegraded":"2023-03-25T04:33:53.335009+0000","last_fullsized":"2023-03-25T04:33:53.335009+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:23.331998+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:33:23.331998+0000","last_peered":"2023-03-25T04:33:23.331998+0000","last_clean":"2023-03-25T04:33:23.331998+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:33:23.331998+0000","last_undegraded":"2023-03-25T04:33:23.331998+0000","last_fullsized":"2023-03-25T04:33:23.331998+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:30.576723+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:33:30.576723+0000","last_peered":"2023-03-25T04:33:30.576723+0000","last_clean":"2023-03-25T04:33:30.576723+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:33:30.576723+0000","last_undegraded":"2023-03-25T04:33:30.576723+0000","last_fullsized":"2023-03-25T04:33:30.576723+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":37,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:38.333447+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:33:38.333447+0000","last_peered":"2023-03-25T04:33:38.333447+0000","last_clean":"2023-03-25T04:33:38.333447+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:33:38.333447+0000","last_undegraded":"2023-03-25T04:33:38.333447+0000","last_fullsized":"2023-03-25T04:33:38.333447+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:38.333467+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:33:38.333467+0000","last_peered":"2023-03-25T04:33:38.333467+0000","last_clean":"2023-03-25T04:33:38.333467+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:33:38.333467+0000","last_undegraded":"2023-03-25T04:33:38.333467+0000","last_fullsized":"2023-03-25T04:33:38.333467+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117102,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:35:49 2023","interfaces":[{"interface":"back","average":{"1min":0.255,"5min":0.26000000000000001,"15min":0.28199999999999997},"min":{"1min":0.17899999999999999,"5min":0.16,"15min":0.16},"max":{"1min":0.36399999999999999,"5min":0.42599999999999999,"15min":0.45100000000000001},"last":0.28399999999999997},{"interface":"front","average":{"1min":0.26000000000000001,"5min":0.26700000000000002,"15min":0.28599999999999998},"min":{"1min":0.188,"5min":0.17999999999999999,"15min":0.17499999999999999},"max":{"1min":0.39100000000000001,"5min":0.45200000000000001,"15min":0.46300000000000002},"last":0.30299999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051693,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:35:42 2023","interfaces":[{"interface":"back","average":{"1min":0.29399999999999998,"5min":0.27200000000000002,"15min":0.25800000000000001},"min":{"1min":0.17299999999999999,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.41099999999999998,"5min":0.42699999999999999,"15min":0.42699999999999999},"last":0.35099999999999998},{"interface":"front","average":{"1min":0.30299999999999999,"5min":0.28399999999999997,"15min":0.27300000000000002},"min":{"1min":0.19700000000000001,"5min":0.192,"15min":0.16600000000000001},"max":{"1min":0.43099999999999999,"5min":0.45800000000000002,"15min":0.45800000000000002},"last":0.32900000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:36:34.556+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:36:34.717+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:36:35.583+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)

Ignoring malparsed backtrace: 2023-03-25T04:36:52.680 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json
, 2023-03-25T04:36:52.701+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:36:53.067 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:37:09.963 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:37:10.273 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all

Ignoring malparsed backtrace: 2023-03-25T04:37:26.856+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:37:27.288 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:37:27.538+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:37:41.903 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":392,"stamp":"2023-03-25T04:37:40.213373+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002012"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":37,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:53.335026+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:33:53.335026+0000","last_peered":"2023-03-25T04:33:53.335026+0000","last_clean":"2023-03-25T04:33:53.335026+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:33:53.335026+0000","last_undegraded":"2023-03-25T04:33:53.335026+0000","last_fullsized":"2023-03-25T04:33:53.335026+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:53.335009+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:33:53.335009+0000","last_peered":"2023-03-25T04:33:53.335009+0000","last_clean":"2023-03-25T04:33:53.335009+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:33:53.335009+0000","last_undegraded":"2023-03-25T04:33:53.335009+0000","last_fullsized":"2023-03-25T04:33:53.335009+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:23.343585+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:35:23.343585+0000","last_peered":"2023-03-25T04:35:23.343585+0000","last_clean":"2023-03-25T04:35:23.343585+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:35:23.343585+0000","last_undegraded":"2023-03-25T04:35:23.343585+0000","last_fullsized":"2023-03-25T04:35:23.343585+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:30.576723+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:33:30.576723+0000","last_peered":"2023-03-25T04:33:30.576723+0000","last_clean":"2023-03-25T04:33:30.576723+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:33:30.576723+0000","last_undegraded":"2023-03-25T04:33:30.576723+0000","last_fullsized":"2023-03-25T04:33:30.576723+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":37,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:38.333447+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:33:38.333447+0000","last_peered":"2023-03-25T04:33:38.333447+0000","last_clean":"2023-03-25T04:33:38.333447+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:33:38.333447+0000","last_undegraded":"2023-03-25T04:33:38.333447+0000","last_fullsized":"2023-03-25T04:33:38.333447+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:38.333467+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:33:38.333467+0000","last_peered":"2023-03-25T04:33:38.333467+0000","last_clean":"2023-03-25T04:33:38.333467+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:33:38.333467+0000","last_undegraded":"2023-03-25T04:33:38.333467+0000","last_fullsized":"2023-03-25T04:33:38.333467+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117119,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:36:50 2023","interfaces":[{"interface":"back","average":{"1min":0.254,"5min":0.25600000000000001,"15min":0.27900000000000003},"min":{"1min":0.20499999999999999,"5min":0.16,"15min":0.16},"max":{"1min":0.371,"5min":0.42599999999999999,"15min":0.45100000000000001},"last":0.23000000000000001},{"interface":"front","average":{"1min":0.26800000000000002,"5min":0.26500000000000001,"15min":0.28399999999999997},"min":{"1min":0.19700000000000001,"5min":0.17999999999999999,"15min":0.17499999999999999},"max":{"1min":0.34699999999999998,"5min":0.45200000000000001,"15min":0.46300000000000002},"last":0.20899999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051710,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:36:44 2023","interfaces":[{"interface":"back","average":{"1min":0.28399999999999997,"5min":0.27700000000000002,"15min":0.26100000000000001},"min":{"1min":0.192,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.40899999999999997,"5min":0.42699999999999999,"15min":0.42699999999999999},"last":0.39700000000000002},{"interface":"front","average":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.27500000000000002},"min":{"1min":0.182,"5min":0.182,"15min":0.16600000000000001},"max":{"1min":0.439,"5min":0.45800000000000002,"15min":0.45800000000000002},"last":0.439}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:37:42.400+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)

Ignoring malparsed backtrace: 2023-03-25T04:38:04.766+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:38:04.922 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:38:05.246 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:38:19.605 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":411,"stamp":"2023-03-25T04:38:18.220966+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002368"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":38,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:36:03.347070+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:36:03.347070+0000","last_peered":"2023-03-25T04:36:03.347070+0000","last_clean":"2023-03-25T04:36:03.347070+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:36:03.347070+0000","last_undegraded":"2023-03-25T04:36:03.347070+0000","last_fullsized":"2023-03-25T04:36:03.347070+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:36:03.347025+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:36:03.347025+0000","last_peered":"2023-03-25T04:36:03.347025+0000","last_clean":"2023-03-25T04:36:03.347025+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:36:03.347025+0000","last_undegraded":"2023-03-25T04:36:03.347025+0000","last_fullsized":"2023-03-25T04:36:03.347025+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:23.343585+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:35:23.343585+0000","last_peered":"2023-03-25T04:35:23.343585+0000","last_clean":"2023-03-25T04:35:23.343585+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:35:23.343585+0000","last_undegraded":"2023-03-25T04:35:23.343585+0000","last_fullsized":"2023-03-25T04:35:23.343585+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:35.587025+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:35:35.587025+0000","last_peered":"2023-03-25T04:35:35.587025+0000","last_clean":"2023-03-25T04:35:35.587025+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:35:35.587025+0000","last_undegraded":"2023-03-25T04:35:35.587025+0000","last_fullsized":"2023-03-25T04:35:35.587025+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":38,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:43.345308+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:35:43.345308+0000","last_peered":"2023-03-25T04:35:43.345308+0000","last_clean":"2023-03-25T04:35:43.345308+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:35:43.345308+0000","last_undegraded":"2023-03-25T04:35:43.345308+0000","last_fullsized":"2023-03-25T04:35:43.345308+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:43.345324+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:35:43.345324+0000","last_peered":"2023-03-25T04:35:43.345324+0000","last_clean":"2023-03-25T04:35:43.345324+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:35:43.345324+0000","last_undegraded":"2023-03-25T04:35:43.345324+0000","last_fullsized":"2023-03-25T04:35:43.345324+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117127,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:37:53 2023","interfaces":[{"interface":"back","average":{"1min":0.25800000000000001,"5min":0.253,"15min":0.27700000000000002},"min":{"1min":0.184,"5min":0.16,"15min":0.16},"max":{"1min":0.44600000000000001,"5min":0.44600000000000001,"15min":0.45100000000000001},"last":0.35199999999999998},{"interface":"front","average":{"1min":0.26900000000000002,"5min":0.26500000000000001,"15min":0.28199999999999997},"min":{"1min":0.192,"5min":0.17999999999999999,"15min":0.17499999999999999},"max":{"1min":0.46400000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"last":0.33200000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051717,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:37:44 2023","interfaces":[{"interface":"back","average":{"1min":0.27300000000000002,"5min":0.28199999999999997,"15min":0.26200000000000001},"min":{"1min":0.17899999999999999,"5min":0.17299999999999999,"15min":0.17000000000000001},"max":{"1min":0.41799999999999998,"5min":0.42099999999999999,"15min":0.42699999999999999},"last":0.28699999999999998},{"interface":"front","average":{"1min":0.29399999999999998,"5min":0.29099999999999998,"15min":0.27700000000000002},"min":{"1min":0.19900000000000001,"5min":0.182,"15min":0.16600000000000001},"max":{"1min":0.441,"5min":0.441,"15min":0.45800000000000002},"last":0.26300000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:38:19.718+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:38:20.315+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)

Ignoring malparsed backtrace: 2023-03-25T04:38:44.283 INFO:teuthology.orchestra.run.smithi070.stdout:
, 2023-03-25T04:38:44.283 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":781,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":16,"modified":"2023-03-25T04:38:40.225321+0000","services":{}},"progress_events":{}}
, 2023-03-25T04:38:44.296 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now

Ignoring malparsed backtrace: 2023-03-25T04:39:02.477+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:39:02.578+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:39:03.524+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:39:20.359 INFO:teuthology.orchestra.run.smithi070.stdout:
, 2023-03-25T04:39:20.359 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":818,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":16,"modified":"2023-03-25T04:38:40.225321+0000","services":{}},"progress_events":{}}
, 2023-03-25T04:39:20.372 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now

Ignoring malparsed backtrace: 2023-03-25T04:39:37.527 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:39:37.541 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":450,"stamp":"2023-03-25T04:39:36.235332+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002086"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":38,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:36:03.347070+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:36:03.347070+0000","last_peered":"2023-03-25T04:36:03.347070+0000","last_clean":"2023-03-25T04:36:03.347070+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:36:03.347070+0000","last_undegraded":"2023-03-25T04:36:03.347070+0000","last_fullsized":"2023-03-25T04:36:03.347070+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:36:03.347025+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:36:03.347025+0000","last_peered":"2023-03-25T04:36:03.347025+0000","last_clean":"2023-03-25T04:36:03.347025+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:36:03.347025+0000","last_undegraded":"2023-03-25T04:36:03.347025+0000","last_fullsized":"2023-03-25T04:36:03.347025+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:23.353778+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:37:23.353778+0000","last_peered":"2023-03-25T04:37:23.353778+0000","last_clean":"2023-03-25T04:37:23.353778+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:37:23.353778+0000","last_undegraded":"2023-03-25T04:37:23.353778+0000","last_fullsized":"2023-03-25T04:37:23.353778+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:35.587025+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:35:35.587025+0000","last_peered":"2023-03-25T04:35:35.587025+0000","last_clean":"2023-03-25T04:35:35.587025+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:35:35.587025+0000","last_undegraded":"2023-03-25T04:35:35.587025+0000","last_fullsized":"2023-03-25T04:35:35.587025+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":38,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:43.345308+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:35:43.345308+0000","last_peered":"2023-03-25T04:35:43.345308+0000","last_clean":"2023-03-25T04:35:43.345308+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:35:43.345308+0000","last_undegraded":"2023-03-25T04:35:43.345308+0000","last_fullsized":"2023-03-25T04:35:43.345308+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:43.345324+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:35:43.345324+0000","last_peered":"2023-03-25T04:35:43.345324+0000","last_clean":"2023-03-25T04:35:43.345324+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:35:43.345324+0000","last_undegraded":"2023-03-25T04:35:43.345324+0000","last_fullsized":"2023-03-25T04:35:43.345324+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117143,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:38:54 2023","interfaces":[{"interface":"back","average":{"1min":0.25800000000000001,"5min":0.25700000000000001,"15min":0.27400000000000002},"min":{"1min":0.192,"5min":0.17399999999999999,"15min":0.16},"max":{"1min":0.39400000000000002,"5min":0.44600000000000001,"15min":0.45100000000000001},"last":0.26500000000000001},{"interface":"front","average":{"1min":0.25900000000000001,"5min":0.26700000000000002,"15min":0.27900000000000003},"min":{"1min":0.19500000000000001,"5min":0.188,"15min":0.17499999999999999},"max":{"1min":0.41499999999999998,"5min":0.46400000000000002,"15min":0.46400000000000002},"last":0.29099999999999998}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051733,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:38:45 2023","interfaces":[{"interface":"back","average":{"1min":0.27300000000000002,"5min":0.28100000000000003,"15min":0.26400000000000001},"min":{"1min":0.19400000000000001,"5min":0.17299999999999999,"15min":0.17000000000000001},"max":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"last":0.27300000000000002},{"interface":"front","average":{"1min":0.28799999999999998,"5min":0.29099999999999998,"15min":0.27800000000000002},"min":{"1min":0.19800000000000001,"5min":0.182,"15min":0.16600000000000001},"max":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"last":0.308}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:39:54.416+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:39:54.455 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:39:54.780 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:40:09.288+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:40:09.389+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:40:10.267+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)

Ignoring malparsed backtrace: 2023-03-25T04:40:28.984 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":476,"stamp":"2023-03-25T04:40:28.245911+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002693"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":39,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:38:13.358968+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:38:13.358968+0000","last_peered":"2023-03-25T04:38:13.358968+0000","last_clean":"2023-03-25T04:38:13.358968+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:38:13.358968+0000","last_undegraded":"2023-03-25T04:38:13.358968+0000","last_fullsized":"2023-03-25T04:38:13.358968+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:38:13.358922+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:38:13.358922+0000","last_peered":"2023-03-25T04:38:13.358922+0000","last_clean":"2023-03-25T04:38:13.358922+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:38:13.358922+0000","last_undegraded":"2023-03-25T04:38:13.358922+0000","last_fullsized":"2023-03-25T04:38:13.358922+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:23.353778+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:37:23.353778+0000","last_peered":"2023-03-25T04:37:23.353778+0000","last_clean":"2023-03-25T04:37:23.353778+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:37:23.353778+0000","last_undegraded":"2023-03-25T04:37:23.353778+0000","last_fullsized":"2023-03-25T04:37:23.353778+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:40.598579+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:37:40.598579+0000","last_peered":"2023-03-25T04:37:40.598579+0000","last_clean":"2023-03-25T04:37:40.598579+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:37:40.598579+0000","last_undegraded":"2023-03-25T04:37:40.598579+0000","last_fullsized":"2023-03-25T04:37:40.598579+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":39,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:48.356344+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:37:48.356344+0000","last_peered":"2023-03-25T04:37:48.356344+0000","last_clean":"2023-03-25T04:37:48.356344+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:37:48.356344+0000","last_undegraded":"2023-03-25T04:37:48.356344+0000","last_fullsized":"2023-03-25T04:37:48.356344+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:48.356365+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:37:48.356365+0000","last_peered":"2023-03-25T04:37:48.356365+0000","last_clean":"2023-03-25T04:37:48.356365+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:37:48.356365+0000","last_undegraded":"2023-03-25T04:37:48.356365+0000","last_fullsized":"2023-03-25T04:37:48.356365+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117153,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:39:54 2023","interfaces":[{"interface":"back","average":{"1min":0.25900000000000001,"5min":0.25600000000000001,"15min":0.27100000000000002},"min":{"1min":0.188,"5min":0.17899999999999999,"15min":0.16},"max":{"1min":0.41999999999999998,"5min":0.44600000000000001,"15min":0.45100000000000001},"last":0.255},{"interface":"front","average":{"1min":0.27500000000000002,"5min":0.26600000000000001,"15min":0.27700000000000002},"min":{"1min":0.17699999999999999,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.45600000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"last":0.26800000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051743,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:39:47 2023","interfaces":[{"interface":"back","average":{"1min":0.26300000000000001,"5min":0.27700000000000002,"15min":0.26500000000000001},"min":{"1min":0.187,"5min":0.17299999999999999,"15min":0.17000000000000001},"max":{"1min":0.433,"5min":0.433,"15min":0.433},"last":0.36599999999999999},{"interface":"front","average":{"1min":0.27100000000000002,"5min":0.28799999999999998,"15min":0.27900000000000003},"min":{"1min":0.20100000000000001,"5min":0.182,"15min":0.16600000000000001},"max":{"1min":0.48599999999999999,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.34999999999999998}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:40:28.986 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean
, 2023-03-25T04:40:28.986 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}

Ignoring malparsed backtrace: 2023-03-25T04:40:47.229+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:40:47.503+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:40:48.251+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)

Ignoring malparsed backtrace: 2023-03-25T04:41:03.267 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:41:03.578 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all

Ignoring malparsed backtrace: 2023-03-25T04:41:21.347 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":502,"stamp":"2023-03-25T04:41:20.256204+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002342"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":39,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:38:13.358968+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:38:13.358968+0000","last_peered":"2023-03-25T04:38:13.358968+0000","last_clean":"2023-03-25T04:38:13.358968+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:38:13.358968+0000","last_undegraded":"2023-03-25T04:38:13.358968+0000","last_fullsized":"2023-03-25T04:38:13.358968+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:38:13.358922+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:38:13.358922+0000","last_peered":"2023-03-25T04:38:13.358922+0000","last_clean":"2023-03-25T04:38:13.358922+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:38:13.358922+0000","last_undegraded":"2023-03-25T04:38:13.358922+0000","last_fullsized":"2023-03-25T04:38:13.358922+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:23.353778+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:37:23.353778+0000","last_peered":"2023-03-25T04:37:23.353778+0000","last_clean":"2023-03-25T04:37:23.353778+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:37:23.353778+0000","last_undegraded":"2023-03-25T04:37:23.353778+0000","last_fullsized":"2023-03-25T04:37:23.353778+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:40.598579+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:37:40.598579+0000","last_peered":"2023-03-25T04:37:40.598579+0000","last_clean":"2023-03-25T04:37:40.598579+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:37:40.598579+0000","last_undegraded":"2023-03-25T04:37:40.598579+0000","last_fullsized":"2023-03-25T04:37:40.598579+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":39,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:48.356344+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:37:48.356344+0000","last_peered":"2023-03-25T04:37:48.356344+0000","last_clean":"2023-03-25T04:37:48.356344+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:37:48.356344+0000","last_undegraded":"2023-03-25T04:37:48.356344+0000","last_fullsized":"2023-03-25T04:37:48.356344+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:48.356365+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:37:48.356365+0000","last_peered":"2023-03-25T04:37:48.356365+0000","last_clean":"2023-03-25T04:37:48.356365+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:37:48.356365+0000","last_undegraded":"2023-03-25T04:37:48.356365+0000","last_fullsized":"2023-03-25T04:37:48.356365+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117163,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:40:56 2023","interfaces":[{"interface":"back","average":{"1min":0.28799999999999998,"5min":0.26300000000000001,"15min":0.27100000000000002},"min":{"1min":0.188,"5min":0.184,"15min":0.16},"max":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.17499999999999999},{"interface":"front","average":{"1min":0.29599999999999999,"5min":0.27300000000000002,"15min":0.27700000000000002},"min":{"1min":0.20599999999999999,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.43099999999999999,"5min":0.46400000000000002,"15min":0.46400000000000002},"last":0.20699999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051754,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:40:52 2023","interfaces":[{"interface":"back","average":{"1min":0.28599999999999998,"5min":0.27500000000000002,"15min":0.26700000000000002},"min":{"1min":0.17899999999999999,"5min":0.17899999999999999,"15min":0.17000000000000001},"max":{"1min":0.44,"5min":0.44,"15min":0.44},"last":0.23699999999999999},{"interface":"front","average":{"1min":0.29799999999999999,"5min":0.28699999999999998,"15min":0.28100000000000003},"min":{"1min":0.20300000000000001,"5min":0.182,"15min":0.16600000000000001},"max":{"1min":0.47599999999999998,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.28000000000000003}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:41:21.349 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean
, 2023-03-25T04:41:21.349 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}

Ignoring malparsed backtrace: 2023-03-25T04:41:39.867+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:41:40.383+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:41:40.905+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:41:58.219 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json
, 2023-03-25T04:41:58.319+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:41:58.644 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:42:18.193 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":530,"stamp":"2023-03-25T04:42:16.266659+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002356"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":39,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:38:13.358968+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:38:13.358968+0000","last_peered":"2023-03-25T04:38:13.358968+0000","last_clean":"2023-03-25T04:38:13.358968+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:38:13.358968+0000","last_undegraded":"2023-03-25T04:38:13.358968+0000","last_fullsized":"2023-03-25T04:38:13.358968+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:38:13.358922+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:38:13.358922+0000","last_peered":"2023-03-25T04:38:13.358922+0000","last_clean":"2023-03-25T04:38:13.358922+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:38:13.358922+0000","last_undegraded":"2023-03-25T04:38:13.358922+0000","last_fullsized":"2023-03-25T04:38:13.358922+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:23.364593+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:39:23.364593+0000","last_peered":"2023-03-25T04:39:23.364593+0000","last_clean":"2023-03-25T04:39:23.364593+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:39:23.364593+0000","last_undegraded":"2023-03-25T04:39:23.364593+0000","last_fullsized":"2023-03-25T04:39:23.364593+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:45.608475+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:39:45.608475+0000","last_peered":"2023-03-25T04:39:45.608475+0000","last_clean":"2023-03-25T04:39:45.608475+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:39:45.608475+0000","last_undegraded":"2023-03-25T04:39:45.608475+0000","last_fullsized":"2023-03-25T04:39:45.608475+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":40,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:53.367398+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:39:53.367398+0000","last_peered":"2023-03-25T04:39:53.367398+0000","last_clean":"2023-03-25T04:39:53.367398+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:39:53.367398+0000","last_undegraded":"2023-03-25T04:39:53.367398+0000","last_fullsized":"2023-03-25T04:39:53.367398+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:53.367424+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:39:53.367424+0000","last_peered":"2023-03-25T04:39:53.367424+0000","last_clean":"2023-03-25T04:39:53.367424+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:39:53.367424+0000","last_undegraded":"2023-03-25T04:39:53.367424+0000","last_fullsized":"2023-03-25T04:39:53.367424+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117175,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:41:58 2023","interfaces":[{"interface":"back","average":{"1min":0.25800000000000001,"5min":0.26400000000000001,"15min":0.26800000000000002},"min":{"1min":0.17499999999999999,"5min":0.17499999999999999,"15min":0.16},"max":{"1min":0.39100000000000001,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.22900000000000001},{"interface":"front","average":{"1min":0.26300000000000001,"5min":0.27200000000000002,"15min":0.27400000000000002},"min":{"1min":0.20599999999999999,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.35699999999999998,"5min":0.46400000000000002,"15min":0.46400000000000002},"last":0.26400000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051765,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:41:52 2023","interfaces":[{"interface":"back","average":{"1min":0.26500000000000001,"5min":0.27200000000000002,"15min":0.26800000000000002},"min":{"1min":0.17000000000000001,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.39200000000000002,"5min":0.44,"15min":0.44},"last":0.23899999999999999},{"interface":"front","average":{"1min":0.28100000000000003,"5min":0.28599999999999998,"15min":0.28199999999999997},"min":{"1min":0.19400000000000001,"5min":0.19400000000000001,"15min":0.16600000000000001},"max":{"1min":0.41399999999999998,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.254}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:42:18.297+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:42:18.958+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:42:35.512 INFO:teuthology.orchestra.run.smithi070.stdout:
, 2023-03-25T04:42:35.512 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":1013,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":17,"modified":"2023-03-25T04:41:54.262744+0000","services":{}},"progress_events":{}}
, 2023-03-25T04:42:35.525 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:42:52.741 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":548,"stamp":"2023-03-25T04:42:52.273601+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002221"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":40,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:40:23.370056+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:40:23.370056+0000","last_peered":"2023-03-25T04:40:23.370056+0000","last_clean":"2023-03-25T04:40:23.370056+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:40:23.370056+0000","last_undegraded":"2023-03-25T04:40:23.370056+0000","last_fullsized":"2023-03-25T04:40:23.370056+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:40:23.370039+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:40:23.370039+0000","last_peered":"2023-03-25T04:40:23.370039+0000","last_clean":"2023-03-25T04:40:23.370039+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:40:23.370039+0000","last_undegraded":"2023-03-25T04:40:23.370039+0000","last_fullsized":"2023-03-25T04:40:23.370039+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:23.364593+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:39:23.364593+0000","last_peered":"2023-03-25T04:39:23.364593+0000","last_clean":"2023-03-25T04:39:23.364593+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:39:23.364593+0000","last_undegraded":"2023-03-25T04:39:23.364593+0000","last_fullsized":"2023-03-25T04:39:23.364593+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:45.608475+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:39:45.608475+0000","last_peered":"2023-03-25T04:39:45.608475+0000","last_clean":"2023-03-25T04:39:45.608475+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:39:45.608475+0000","last_undegraded":"2023-03-25T04:39:45.608475+0000","last_fullsized":"2023-03-25T04:39:45.608475+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":40,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:53.367398+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:39:53.367398+0000","last_peered":"2023-03-25T04:39:53.367398+0000","last_clean":"2023-03-25T04:39:53.367398+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:39:53.367398+0000","last_undegraded":"2023-03-25T04:39:53.367398+0000","last_fullsized":"2023-03-25T04:39:53.367398+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:53.367424+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:39:53.367424+0000","last_peered":"2023-03-25T04:39:53.367424+0000","last_clean":"2023-03-25T04:39:53.367424+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:39:53.367424+0000","last_undegraded":"2023-03-25T04:39:53.367424+0000","last_fullsized":"2023-03-25T04:39:53.367424+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117182,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:41:58 2023","interfaces":[{"interface":"back","average":{"1min":0.25800000000000001,"5min":0.26400000000000001,"15min":0.26800000000000002},"min":{"1min":0.17499999999999999,"5min":0.17499999999999999,"15min":0.16},"max":{"1min":0.39100000000000001,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.19600000000000001},{"interface":"front","average":{"1min":0.26300000000000001,"5min":0.27200000000000002,"15min":0.27400000000000002},"min":{"1min":0.20599999999999999,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.35699999999999998,"5min":0.46400000000000002,"15min":0.46400000000000002},"last":0.219}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051773,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:41:52 2023","interfaces":[{"interface":"back","average":{"1min":0.26500000000000001,"5min":0.27200000000000002,"15min":0.26800000000000002},"min":{"1min":0.17000000000000001,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.39200000000000002,"5min":0.44,"15min":0.44},"last":0.19},{"interface":"front","average":{"1min":0.28100000000000003,"5min":0.28599999999999998,"15min":0.28199999999999997},"min":{"1min":0.19400000000000001,"5min":0.19400000000000001,"15min":0.16600000000000001},"max":{"1min":0.41399999999999998,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.17599999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:42:52.742 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean

Ignoring malparsed backtrace: 2023-03-25T04:43:09.973 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:43:09.986 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":556,"stamp":"2023-03-25T04:43:08.276556+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002359"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":40,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:40:23.370056+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:40:23.370056+0000","last_peered":"2023-03-25T04:40:23.370056+0000","last_clean":"2023-03-25T04:40:23.370056+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:40:23.370056+0000","last_undegraded":"2023-03-25T04:40:23.370056+0000","last_fullsized":"2023-03-25T04:40:23.370056+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:40:23.370039+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:40:23.370039+0000","last_peered":"2023-03-25T04:40:23.370039+0000","last_clean":"2023-03-25T04:40:23.370039+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:40:23.370039+0000","last_undegraded":"2023-03-25T04:40:23.370039+0000","last_fullsized":"2023-03-25T04:40:23.370039+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:23.364593+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:39:23.364593+0000","last_peered":"2023-03-25T04:39:23.364593+0000","last_clean":"2023-03-25T04:39:23.364593+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:39:23.364593+0000","last_undegraded":"2023-03-25T04:39:23.364593+0000","last_fullsized":"2023-03-25T04:39:23.364593+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:45.608475+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:39:45.608475+0000","last_peered":"2023-03-25T04:39:45.608475+0000","last_clean":"2023-03-25T04:39:45.608475+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:39:45.608475+0000","last_undegraded":"2023-03-25T04:39:45.608475+0000","last_fullsized":"2023-03-25T04:39:45.608475+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":40,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:53.367398+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:39:53.367398+0000","last_peered":"2023-03-25T04:39:53.367398+0000","last_clean":"2023-03-25T04:39:53.367398+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:39:53.367398+0000","last_undegraded":"2023-03-25T04:39:53.367398+0000","last_fullsized":"2023-03-25T04:39:53.367398+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:53.367424+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:39:53.367424+0000","last_peered":"2023-03-25T04:39:53.367424+0000","last_clean":"2023-03-25T04:39:53.367424+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:39:53.367424+0000","last_undegraded":"2023-03-25T04:39:53.367424+0000","last_fullsized":"2023-03-25T04:39:53.367424+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117185,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:43:00 2023","interfaces":[{"interface":"back","average":{"1min":0.27400000000000002,"5min":0.26700000000000002,"15min":0.26600000000000001},"min":{"1min":0.19600000000000001,"5min":0.17499999999999999,"15min":0.16},"max":{"1min":0.42999999999999999,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.27800000000000002},{"interface":"front","average":{"1min":0.28199999999999997,"5min":0.27500000000000002,"15min":0.27300000000000002},"min":{"1min":0.185,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.47199999999999998,"5min":0.47199999999999998,"15min":0.47199999999999998},"last":0.247}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051776,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:42:54 2023","interfaces":[{"interface":"back","average":{"1min":0.26300000000000001,"5min":0.27000000000000002,"15min":0.26900000000000002},"min":{"1min":0.17399999999999999,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.438,"5min":0.44,"15min":0.44},"last":0.20100000000000001},{"interface":"front","average":{"1min":0.27400000000000002,"5min":0.28199999999999997,"15min":0.28299999999999997},"min":{"1min":0.17599999999999999,"5min":0.17599999999999999,"15min":0.16600000000000001},"max":{"1min":0.41599999999999998,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.23499999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:43:25.042+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:43:25.411+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:43:26.013+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:43:47.937+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:43:48.534+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:43:48.889+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:44:09.318 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:44:09.331 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":586,"stamp":"2023-03-25T04:44:08.288933+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002697"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":40,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:40:23.370056+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:40:23.370056+0000","last_peered":"2023-03-25T04:40:23.370056+0000","last_clean":"2023-03-25T04:40:23.370056+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:40:23.370056+0000","last_undegraded":"2023-03-25T04:40:23.370056+0000","last_fullsized":"2023-03-25T04:40:23.370056+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:40:23.370039+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:40:23.370039+0000","last_peered":"2023-03-25T04:40:23.370039+0000","last_clean":"2023-03-25T04:40:23.370039+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:40:23.370039+0000","last_undegraded":"2023-03-25T04:40:23.370039+0000","last_fullsized":"2023-03-25T04:40:23.370039+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:23.375106+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:41:23.375106+0000","last_peered":"2023-03-25T04:41:23.375106+0000","last_clean":"2023-03-25T04:41:23.375106+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:41:23.375106+0000","last_undegraded":"2023-03-25T04:41:23.375106+0000","last_fullsized":"2023-03-25T04:41:23.375106+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:50.618930+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:41:50.618930+0000","last_peered":"2023-03-25T04:41:50.618930+0000","last_clean":"2023-03-25T04:41:50.618930+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:41:50.618930+0000","last_undegraded":"2023-03-25T04:41:50.618930+0000","last_fullsized":"2023-03-25T04:41:50.618930+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":41,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:58.377821+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:41:58.377821+0000","last_peered":"2023-03-25T04:41:58.377821+0000","last_clean":"2023-03-25T04:41:58.377821+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:41:58.377821+0000","last_undegraded":"2023-03-25T04:41:58.377821+0000","last_fullsized":"2023-03-25T04:41:58.377821+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:58.377836+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:41:58.377836+0000","last_peered":"2023-03-25T04:41:58.377836+0000","last_clean":"2023-03-25T04:41:58.377836+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:41:58.377836+0000","last_undegraded":"2023-03-25T04:41:58.377836+0000","last_fullsized":"2023-03-25T04:41:58.377836+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117197,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:44:01 2023","interfaces":[{"interface":"back","average":{"1min":0.27200000000000002,"5min":0.27000000000000002,"15min":0.26400000000000001},"min":{"1min":0.20399999999999999,"5min":0.17499999999999999,"15min":0.16},"max":{"1min":0.38400000000000001,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.245},{"interface":"front","average":{"1min":0.27600000000000002,"5min":0.27800000000000002,"15min":0.27100000000000002},"min":{"1min":0.23100000000000001,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.40400000000000003,"5min":0.47199999999999998,"15min":0.47199999999999998},"last":0.27500000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051788,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:43:55 2023","interfaces":[{"interface":"back","average":{"1min":0.26100000000000001,"5min":0.26700000000000002,"15min":0.27000000000000002},"min":{"1min":0.17799999999999999,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.374,"5min":0.44,"15min":0.44},"last":0.30299999999999999},{"interface":"front","average":{"1min":0.28299999999999997,"5min":0.28100000000000003,"15min":0.28399999999999997},"min":{"1min":0.20899999999999999,"5min":0.17599999999999999,"15min":0.16600000000000001},"max":{"1min":0.40100000000000002,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.27800000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:44:28.923 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":1126,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":19,"modified":"2023-03-25T04:43:20.278959+0000","services":{}},"progress_events":{}}
, 2023-03-25T04:44:28.936 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now
, 2023-03-25T04:44:28.937 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace: 2023-03-25T04:44:48.786 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:44:48.800 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":606,"stamp":"2023-03-25T04:44:48.296805+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002182"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":41,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:42:34.382196+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:42:34.382196+0000","last_peered":"2023-03-25T04:42:34.382196+0000","last_clean":"2023-03-25T04:42:34.382196+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:42:34.382196+0000","last_undegraded":"2023-03-25T04:42:34.382196+0000","last_fullsized":"2023-03-25T04:42:34.382196+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:42:29.381744+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:42:29.381744+0000","last_peered":"2023-03-25T04:42:29.381744+0000","last_clean":"2023-03-25T04:42:29.381744+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:42:29.381744+0000","last_undegraded":"2023-03-25T04:42:29.381744+0000","last_fullsized":"2023-03-25T04:42:29.381744+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:23.375106+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:41:23.375106+0000","last_peered":"2023-03-25T04:41:23.375106+0000","last_clean":"2023-03-25T04:41:23.375106+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:41:23.375106+0000","last_undegraded":"2023-03-25T04:41:23.375106+0000","last_fullsized":"2023-03-25T04:41:23.375106+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:50.618930+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:41:50.618930+0000","last_peered":"2023-03-25T04:41:50.618930+0000","last_clean":"2023-03-25T04:41:50.618930+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:41:50.618930+0000","last_undegraded":"2023-03-25T04:41:50.618930+0000","last_fullsized":"2023-03-25T04:41:50.618930+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":41,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:58.377821+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:41:58.377821+0000","last_peered":"2023-03-25T04:41:58.377821+0000","last_clean":"2023-03-25T04:41:58.377821+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:41:58.377821+0000","last_undegraded":"2023-03-25T04:41:58.377821+0000","last_fullsized":"2023-03-25T04:41:58.377821+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:58.377836+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:41:58.377836+0000","last_peered":"2023-03-25T04:41:58.377836+0000","last_clean":"2023-03-25T04:41:58.377836+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:41:58.377836+0000","last_undegraded":"2023-03-25T04:41:58.377836+0000","last_fullsized":"2023-03-25T04:41:58.377836+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117206,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:44:01 2023","interfaces":[{"interface":"back","average":{"1min":0.27200000000000002,"5min":0.27000000000000002,"15min":0.26400000000000001},"min":{"1min":0.20399999999999999,"5min":0.17499999999999999,"15min":0.16},"max":{"1min":0.38400000000000001,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.216},{"interface":"front","average":{"1min":0.27600000000000002,"5min":0.27800000000000002,"15min":0.27100000000000002},"min":{"1min":0.23100000000000001,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.40400000000000003,"5min":0.47199999999999998,"15min":0.47199999999999998},"last":0.23999999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051796,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:43:55 2023","interfaces":[{"interface":"back","average":{"1min":0.26100000000000001,"5min":0.26700000000000002,"15min":0.27000000000000002},"min":{"1min":0.17799999999999999,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.374,"5min":0.44,"15min":0.44},"last":0.25600000000000001},{"interface":"front","average":{"1min":0.28299999999999997,"5min":0.28100000000000003,"15min":0.28399999999999997},"min":{"1min":0.20899999999999999,"5min":0.17599999999999999,"15min":0.16600000000000001},"max":{"1min":0.40100000000000002,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.28000000000000003}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:45:05.762 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:45:06.073 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all

Ignoring malparsed backtrace: 2023-03-25T04:45:25.658 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":624,"stamp":"2023-03-25T04:45:24.304294+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002366"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":41,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:42:34.382196+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:42:34.382196+0000","last_peered":"2023-03-25T04:42:34.382196+0000","last_clean":"2023-03-25T04:42:34.382196+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:42:34.382196+0000","last_undegraded":"2023-03-25T04:42:34.382196+0000","last_fullsized":"2023-03-25T04:42:34.382196+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:42:29.381744+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:42:29.381744+0000","last_peered":"2023-03-25T04:42:29.381744+0000","last_clean":"2023-03-25T04:42:29.381744+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:42:29.381744+0000","last_undegraded":"2023-03-25T04:42:29.381744+0000","last_fullsized":"2023-03-25T04:42:29.381744+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:23.375106+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:41:23.375106+0000","last_peered":"2023-03-25T04:41:23.375106+0000","last_clean":"2023-03-25T04:41:23.375106+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:41:23.375106+0000","last_undegraded":"2023-03-25T04:41:23.375106+0000","last_fullsized":"2023-03-25T04:41:23.375106+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:50.618930+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:41:50.618930+0000","last_peered":"2023-03-25T04:41:50.618930+0000","last_clean":"2023-03-25T04:41:50.618930+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:41:50.618930+0000","last_undegraded":"2023-03-25T04:41:50.618930+0000","last_fullsized":"2023-03-25T04:41:50.618930+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":41,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:58.377821+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:41:58.377821+0000","last_peered":"2023-03-25T04:41:58.377821+0000","last_clean":"2023-03-25T04:41:58.377821+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:41:58.377821+0000","last_undegraded":"2023-03-25T04:41:58.377821+0000","last_fullsized":"2023-03-25T04:41:58.377821+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:58.377836+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:41:58.377836+0000","last_peered":"2023-03-25T04:41:58.377836+0000","last_clean":"2023-03-25T04:41:58.377836+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:41:58.377836+0000","last_undegraded":"2023-03-25T04:41:58.377836+0000","last_fullsized":"2023-03-25T04:41:58.377836+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117213,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:45:01 2023","interfaces":[{"interface":"back","average":{"1min":0.251,"5min":0.26800000000000002,"15min":0.26400000000000001},"min":{"1min":0.183,"5min":0.17499999999999999,"15min":0.16},"max":{"1min":0.41499999999999998,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.26900000000000002},{"interface":"front","average":{"1min":0.28000000000000003,"5min":0.27900000000000003,"15min":0.27100000000000002},"min":{"1min":0.20999999999999999,"5min":0.185,"15min":0.17499999999999999},"max":{"1min":0.44400000000000001,"5min":0.47199999999999998,"15min":0.47199999999999998},"last":0.27900000000000003}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051803,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:44:55 2023","interfaces":[{"interface":"back","average":{"1min":0.252,"5min":0.26500000000000001,"15min":0.26900000000000002},"min":{"1min":0.183,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.39700000000000002,"5min":0.44,"15min":0.44},"last":0.20699999999999999},{"interface":"front","average":{"1min":0.27000000000000002,"5min":0.28100000000000003,"15min":0.28299999999999997},"min":{"1min":0.20300000000000001,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":0.42099999999999999,"5min":0.47599999999999998,"15min":0.48599999999999999},"last":0.23599999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:45:25.659 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:45:25.969 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:45:42.735+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:45:42.744+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:45:42.897 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace: 2023-03-25T04:46:00.183 INFO:teuthology.orchestra.run.smithi070.stdout:
, 2023-03-25T04:46:00.183 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":1217,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":20,"modified":"2023-03-25T04:45:42.307700+0000","services":{"mgr":{"daemons":{"summary":"","x":{"start_epoch":0,"start_stamp":"0.000000","gid":0,"addr":"(unrecognized address family 0)/0","metadata":{},"task_status":{}}}}}},"progress_events":{}}
, 2023-03-25T04:46:00.197 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now

Ignoring malparsed backtrace: 2023-03-25T04:46:19.910+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:46:20.039 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all

Ignoring malparsed backtrace: 2023-03-25T04:46:39.611 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}
, 2023-03-25T04:46:39.611 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json
, 2023-03-25T04:46:39.613+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:46:58.485+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:46:58.764+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:46:59.519+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:47:14.132 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:47:14.146 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":678,"stamp":"2023-03-25T04:47:12.325044+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002571"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":42,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:44.393426+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:44:44.393426+0000","last_peered":"2023-03-25T04:44:44.393426+0000","last_clean":"2023-03-25T04:44:44.393426+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:44:44.393426+0000","last_undegraded":"2023-03-25T04:44:44.393426+0000","last_fullsized":"2023-03-25T04:44:44.393426+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:39.393044+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:44:39.393044+0000","last_peered":"2023-03-25T04:44:39.393044+0000","last_clean":"2023-03-25T04:44:39.393044+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:44:39.393044+0000","last_undegraded":"2023-03-25T04:44:39.393044+0000","last_fullsized":"2023-03-25T04:44:39.393044+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:43:24.386177+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:43:24.386177+0000","last_peered":"2023-03-25T04:43:24.386177+0000","last_clean":"2023-03-25T04:43:24.386177+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:43:24.386177+0000","last_undegraded":"2023-03-25T04:43:24.386177+0000","last_fullsized":"2023-03-25T04:43:24.386177+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:43:55.629798+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:43:55.629798+0000","last_peered":"2023-03-25T04:43:55.629798+0000","last_clean":"2023-03-25T04:43:55.629798+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:43:55.629798+0000","last_undegraded":"2023-03-25T04:43:55.629798+0000","last_fullsized":"2023-03-25T04:43:55.629798+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":42,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:04.389904+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:44:04.389904+0000","last_peered":"2023-03-25T04:44:04.389904+0000","last_clean":"2023-03-25T04:44:04.389904+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:44:04.389904+0000","last_undegraded":"2023-03-25T04:44:04.389904+0000","last_fullsized":"2023-03-25T04:44:04.389904+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:04.389923+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:44:04.389923+0000","last_peered":"2023-03-25T04:44:04.389923+0000","last_clean":"2023-03-25T04:44:04.389923+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:44:04.389923+0000","last_undegraded":"2023-03-25T04:44:04.389923+0000","last_fullsized":"2023-03-25T04:44:04.389923+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117235,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:47:05 2023","interfaces":[{"interface":"back","average":{"1min":0.27800000000000002,"5min":0.27000000000000002,"15min":0.26300000000000001},"min":{"1min":0.20100000000000001,"5min":0.16900000000000001,"15min":0.16},"max":{"1min":0.39200000000000002,"5min":0.42999999999999999,"15min":0.45600000000000002},"last":0.379},{"interface":"front","average":{"1min":0.28399999999999997,"5min":0.28100000000000003,"15min":0.27200000000000002},"min":{"1min":0.17799999999999999,"5min":0.17799999999999999,"15min":0.17699999999999999},"max":{"1min":0.42599999999999999,"5min":0.47199999999999998,"15min":0.47199999999999998},"last":0.39800000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051825,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:47:01 2023","interfaces":[{"interface":"back","average":{"1min":0.27300000000000002,"5min":0.26100000000000001,"15min":0.27000000000000002},"min":{"1min":0.186,"5min":0.17399999999999999,"15min":0.17000000000000001},"max":{"1min":0.40400000000000003,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.23899999999999999},{"interface":"front","average":{"1min":0.28699999999999998,"5min":0.27500000000000002,"15min":0.28299999999999997},"min":{"1min":0.186,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.48599999999999999},"last":0.19600000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:47:31.198 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json
, 2023-03-25T04:47:31.411+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:47:31.789 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:47:49.509 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":696,"stamp":"2023-03-25T04:47:48.332093+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002697"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":42,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:44.393426+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:44:44.393426+0000","last_peered":"2023-03-25T04:44:44.393426+0000","last_clean":"2023-03-25T04:44:44.393426+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:44:44.393426+0000","last_undegraded":"2023-03-25T04:44:44.393426+0000","last_fullsized":"2023-03-25T04:44:44.393426+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:39.393044+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:44:39.393044+0000","last_peered":"2023-03-25T04:44:39.393044+0000","last_clean":"2023-03-25T04:44:39.393044+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:44:39.393044+0000","last_undegraded":"2023-03-25T04:44:39.393044+0000","last_fullsized":"2023-03-25T04:44:39.393044+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:45:24.396717+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:45:24.396717+0000","last_peered":"2023-03-25T04:45:24.396717+0000","last_clean":"2023-03-25T04:45:24.396717+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:45:24.396717+0000","last_undegraded":"2023-03-25T04:45:24.396717+0000","last_fullsized":"2023-03-25T04:45:24.396717+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:43:55.629798+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:43:55.629798+0000","last_peered":"2023-03-25T04:43:55.629798+0000","last_clean":"2023-03-25T04:43:55.629798+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:43:55.629798+0000","last_undegraded":"2023-03-25T04:43:55.629798+0000","last_fullsized":"2023-03-25T04:43:55.629798+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":42,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:04.389904+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:44:04.389904+0000","last_peered":"2023-03-25T04:44:04.389904+0000","last_clean":"2023-03-25T04:44:04.389904+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:44:04.389904+0000","last_undegraded":"2023-03-25T04:44:04.389904+0000","last_fullsized":"2023-03-25T04:44:04.389904+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:04.389923+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:44:04.389923+0000","last_peered":"2023-03-25T04:44:04.389923+0000","last_clean":"2023-03-25T04:44:04.389923+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:44:04.389923+0000","last_undegraded":"2023-03-25T04:44:04.389923+0000","last_fullsized":"2023-03-25T04:44:04.389923+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117242,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:47:05 2023","interfaces":[{"interface":"back","average":{"1min":0.27800000000000002,"5min":0.27000000000000002,"15min":0.26300000000000001},"min":{"1min":0.20100000000000001,"5min":0.16900000000000001,"15min":0.16},"max":{"1min":0.39200000000000002,"5min":0.42999999999999999,"15min":0.45600000000000002},"last":0.40500000000000003},{"interface":"front","average":{"1min":0.28399999999999997,"5min":0.28100000000000003,"15min":0.27200000000000002},"min":{"1min":0.17799999999999999,"5min":0.17799999999999999,"15min":0.17699999999999999},"max":{"1min":0.42599999999999999,"5min":0.47199999999999998,"15min":0.47199999999999998},"last":0.42399999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051834,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:47:01 2023","interfaces":[{"interface":"back","average":{"1min":0.27300000000000002,"5min":0.26100000000000001,"15min":0.27000000000000002},"min":{"1min":0.186,"5min":0.17399999999999999,"15min":0.17000000000000001},"max":{"1min":0.40400000000000003,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.25600000000000001},{"interface":"front","average":{"1min":0.28699999999999998,"5min":0.27500000000000002,"15min":0.28299999999999997},"min":{"1min":0.186,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.48599999999999999},"last":0.22800000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:47:49.510 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace:     ctx.managers[config['cluster']].wait_for_clean()
,   File "/home/teuthworker/src/github.com_ceph_ceph-c_511015c23eef0684a64171b710e432448df2b7c7/qa/tasks/ceph_manager.py", line 2747, in wait_for_clean
,     assert time.time() - start < timeout, \

Ignoring malparsed backtrace:  ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
,  1: /lib64/libpthread.so.0(+0x12cf0) [0x7f1453c2fcf0]
,  2: __pthread_mutex_lock()

Ignoring malparsed backtrace: 2023-03-25T04:28:05.172 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":142,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":5,"modified":"2023-03-25T04:27:12.092098+0000","services":{}},"progress_events":{}}
, 2023-03-25T04:28:05.185 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now
, 2023-03-25T04:28:05.186 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace: 2023-03-25T04:28:28.360 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":116,"stamp":"2023-03-25T04:28:28.108007+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002578"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.314661+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:27:45.314661+0000","last_peered":"2023-03-25T04:27:45.314661+0000","last_clean":"2023-03-25T04:27:45.314661+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:27:45.314661+0000","last_undegraded":"2023-03-25T04:27:45.314661+0000","last_fullsized":"2023-03-25T04:27:45.314661+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":25,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.310745+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:27:44.310745+0000","last_peered":"2023-03-25T04:27:44.310745+0000","last_clean":"2023-03-25T04:27:44.310745+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:27:44.310745+0000","last_undegraded":"2023-03-25T04:27:44.310745+0000","last_fullsized":"2023-03-25T04:27:44.310745+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":25,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.310650+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:27:44.310650+0000","last_peered":"2023-03-25T04:27:44.310650+0000","last_clean":"2023-03-25T04:27:44.310650+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:27:44.310650+0000","last_undegraded":"2023-03-25T04:27:44.310650+0000","last_fullsized":"2023-03-25T04:27:44.310650+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":26,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.312962+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:27:44.312962+0000","last_peered":"2023-03-25T04:27:44.312962+0000","last_clean":"2023-03-25T04:27:44.312962+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:27:44.312962+0000","last_undegraded":"2023-03-25T04:27:44.312962+0000","last_fullsized":"2023-03-25T04:27:44.312962+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315064+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:27:45.315064+0000","last_peered":"2023-03-25T04:27:45.315064+0000","last_clean":"2023-03-25T04:27:45.315064+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:27:45.315064+0000","last_undegraded":"2023-03-25T04:27:45.315064+0000","last_fullsized":"2023-03-25T04:27:45.315064+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":25,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.310707+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:27:44.310707+0000","last_peered":"2023-03-25T04:27:44.310707+0000","last_clean":"2023-03-25T04:27:44.310707+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:27:44.310707+0000","last_undegraded":"2023-03-25T04:27:44.310707+0000","last_fullsized":"2023-03-25T04:27:44.310707+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117007,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.375}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051597,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.186}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:28:28.362 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean
, 2023-03-25T04:28:28.362 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:28:45.655 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":124,"stamp":"2023-03-25T04:28:44.111265+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002492"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.314661+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:27:45.314661+0000","last_peered":"2023-03-25T04:27:45.314661+0000","last_clean":"2023-03-25T04:27:45.314661+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:27:45.314661+0000","last_undegraded":"2023-03-25T04:27:45.314661+0000","last_fullsized":"2023-03-25T04:27:45.314661+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":25,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.310745+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:27:44.310745+0000","last_peered":"2023-03-25T04:27:44.310745+0000","last_clean":"2023-03-25T04:27:44.310745+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:27:44.310745+0000","last_undegraded":"2023-03-25T04:27:44.310745+0000","last_fullsized":"2023-03-25T04:27:44.310745+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":25,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.310650+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:27:44.310650+0000","last_peered":"2023-03-25T04:27:44.310650+0000","last_clean":"2023-03-25T04:27:44.310650+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:27:44.310650+0000","last_undegraded":"2023-03-25T04:27:44.310650+0000","last_fullsized":"2023-03-25T04:27:44.310650+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":26,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.312962+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:27:44.312962+0000","last_peered":"2023-03-25T04:27:44.312962+0000","last_clean":"2023-03-25T04:27:44.312962+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:27:44.312962+0000","last_undegraded":"2023-03-25T04:27:44.312962+0000","last_fullsized":"2023-03-25T04:27:44.312962+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315064+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:27:45.315064+0000","last_peered":"2023-03-25T04:27:45.315064+0000","last_clean":"2023-03-25T04:27:45.315064+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:27:45.315064+0000","last_undegraded":"2023-03-25T04:27:45.315064+0000","last_fullsized":"2023-03-25T04:27:45.315064+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":25,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.310707+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:27:44.310707+0000","last_peered":"2023-03-25T04:27:44.310707+0000","last_clean":"2023-03-25T04:27:44.310707+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:27:44.310707+0000","last_undegraded":"2023-03-25T04:27:44.310707+0000","last_fullsized":"2023-03-25T04:27:44.310707+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117011,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:28:31 2023","interfaces":[{"interface":"back","average":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.36899999999999999},{"interface":"front","average":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"min":{"1min":0.19,"5min":0.19,"15min":0.19},"max":{"1min":0.46300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.39800000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051601,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:28:24 2023","interfaces":[{"interface":"back","average":{"1min":0.249,"5min":0.249,"15min":0.249},"min":{"1min":0.17699999999999999,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"last":0.254},{"interface":"front","average":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"min":{"1min":0.20200000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"last":0.312}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:28:45.656 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace: 2023-03-25T04:29:05.171 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}
, 2023-03-25T04:29:05.171 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json
, 2023-03-25T04:29:05.561 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:29:22.465 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:29:22.773 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all

Ignoring malparsed backtrace: 2023-03-25T04:29:42.405 INFO:teuthology.orchestra.run.smithi070.stdout:
, 2023-03-25T04:29:42.406 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":240,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":9,"modified":"2023-03-25T04:29:32.120358+0000","services":{"mon":{"daemons":{"summary":"","a":{"start_epoch":0,"start_stamp":"0.000000","gid":0,"addr":"(unrecognized address family 0)/0","metadata":{},"task_status":{}}}}}},"progress_events":{}}
, 2023-03-25T04:29:42.419 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:29:59.627 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":161,"stamp":"2023-03-25T04:29:58.125186+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002398"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315468+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:27:45.315468+0000","last_peered":"2023-03-25T04:27:45.315468+0000","last_clean":"2023-03-25T04:27:45.315468+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:27:45.315468+0000","last_undegraded":"2023-03-25T04:27:45.315468+0000","last_fullsized":"2023-03-25T04:27:45.315468+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312845+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:27:45.312845+0000","last_peered":"2023-03-25T04:27:45.312845+0000","last_clean":"2023-03-25T04:27:45.312845+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:27:45.312845+0000","last_undegraded":"2023-03-25T04:27:45.312845+0000","last_fullsized":"2023-03-25T04:27:45.312845+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312882+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:27:45.312882+0000","last_peered":"2023-03-25T04:27:45.312882+0000","last_clean":"2023-03-25T04:27:45.312882+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:27:45.312882+0000","last_undegraded":"2023-03-25T04:27:45.312882+0000","last_fullsized":"2023-03-25T04:27:45.312882+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.311730+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:27:45.311730+0000","last_peered":"2023-03-25T04:27:45.311730+0000","last_clean":"2023-03-25T04:27:45.311730+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:27:45.311730+0000","last_undegraded":"2023-03-25T04:27:45.311730+0000","last_fullsized":"2023-03-25T04:27:45.311730+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315735+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:27:45.315735+0000","last_peered":"2023-03-25T04:27:45.315735+0000","last_clean":"2023-03-25T04:27:45.315735+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:27:45.315735+0000","last_undegraded":"2023-03-25T04:27:45.315735+0000","last_fullsized":"2023-03-25T04:27:45.315735+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312833+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:27:45.312833+0000","last_peered":"2023-03-25T04:27:45.312833+0000","last_clean":"2023-03-25T04:27:45.312833+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:27:45.312833+0000","last_undegraded":"2023-03-25T04:27:45.312833+0000","last_fullsized":"2023-03-25T04:27:45.312833+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117026,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:29:33 2023","interfaces":[{"interface":"back","average":{"1min":0.26300000000000001,"5min":0.29099999999999998,"15min":0.29599999999999999},"min":{"1min":0.191,"5min":0.191,"15min":0.191},"max":{"1min":0.371,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.20499999999999999},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.29599999999999999,"15min":0.29999999999999999},"min":{"1min":0.21099999999999999,"5min":0.19,"15min":0.19},"max":{"1min":0.39800000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.19500000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051616,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:29:29 2023","interfaces":[{"interface":"back","average":{"1min":0.27200000000000002,"5min":0.253,"15min":0.25},"min":{"1min":0.183,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"last":0.26400000000000001},{"interface":"front","average":{"1min":0.28499999999999998,"5min":0.26800000000000002,"15min":0.26500000000000001},"min":{"1min":0.16600000000000001,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"last":0.29899999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:29:59.629 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean

Ignoring malparsed backtrace: 2023-03-25T04:30:16.889 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:30:16.903 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":170,"stamp":"2023-03-25T04:30:16.128427+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002222"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315468+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:27:45.315468+0000","last_peered":"2023-03-25T04:27:45.315468+0000","last_clean":"2023-03-25T04:27:45.315468+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:27:45.315468+0000","last_undegraded":"2023-03-25T04:27:45.315468+0000","last_fullsized":"2023-03-25T04:27:45.315468+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312845+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:27:45.312845+0000","last_peered":"2023-03-25T04:27:45.312845+0000","last_clean":"2023-03-25T04:27:45.312845+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:27:45.312845+0000","last_undegraded":"2023-03-25T04:27:45.312845+0000","last_fullsized":"2023-03-25T04:27:45.312845+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312882+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:27:45.312882+0000","last_peered":"2023-03-25T04:27:45.312882+0000","last_clean":"2023-03-25T04:27:45.312882+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:27:45.312882+0000","last_undegraded":"2023-03-25T04:27:45.312882+0000","last_fullsized":"2023-03-25T04:27:45.312882+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.311730+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:27:45.311730+0000","last_peered":"2023-03-25T04:27:45.311730+0000","last_clean":"2023-03-25T04:27:45.311730+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:27:45.311730+0000","last_undegraded":"2023-03-25T04:27:45.311730+0000","last_fullsized":"2023-03-25T04:27:45.311730+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315735+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:27:45.315735+0000","last_peered":"2023-03-25T04:27:45.315735+0000","last_clean":"2023-03-25T04:27:45.315735+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:27:45.315735+0000","last_undegraded":"2023-03-25T04:27:45.315735+0000","last_fullsized":"2023-03-25T04:27:45.315735+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312833+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:27:45.312833+0000","last_peered":"2023-03-25T04:27:45.312833+0000","last_clean":"2023-03-25T04:27:45.312833+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:27:45.312833+0000","last_undegraded":"2023-03-25T04:27:45.312833+0000","last_fullsized":"2023-03-25T04:27:45.312833+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117030,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:29:33 2023","interfaces":[{"interface":"back","average":{"1min":0.26300000000000001,"5min":0.29099999999999998,"15min":0.29599999999999999},"min":{"1min":0.191,"5min":0.191,"15min":0.191},"max":{"1min":0.371,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.245},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.29599999999999999,"15min":0.29999999999999999},"min":{"1min":0.21099999999999999,"5min":0.19,"15min":0.19},"max":{"1min":0.39800000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.20899999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051620,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:29:29 2023","interfaces":[{"interface":"back","average":{"1min":0.27200000000000002,"5min":0.253,"15min":0.25},"min":{"1min":0.183,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"last":0.19400000000000001},{"interface":"front","average":{"1min":0.28499999999999998,"5min":0.26800000000000002,"15min":0.26500000000000001},"min":{"1min":0.16600000000000001,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"last":0.23599999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:30:31.984+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:30:32.201+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:30:32.981+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)

Ignoring malparsed backtrace: 2023-03-25T04:30:48.449 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now
, 2023-03-25T04:30:48.450 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:30:48.760 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:31:08.342 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":196,"stamp":"2023-03-25T04:31:08.139018+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002431"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315468+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:27:45.315468+0000","last_peered":"2023-03-25T04:27:45.315468+0000","last_clean":"2023-03-25T04:27:45.315468+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:27:45.315468+0000","last_undegraded":"2023-03-25T04:27:45.315468+0000","last_fullsized":"2023-03-25T04:27:45.315468+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312845+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:27:45.312845+0000","last_peered":"2023-03-25T04:27:45.312845+0000","last_clean":"2023-03-25T04:27:45.312845+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:27:45.312845+0000","last_undegraded":"2023-03-25T04:27:45.312845+0000","last_fullsized":"2023-03-25T04:27:45.312845+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312882+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:27:45.312882+0000","last_peered":"2023-03-25T04:27:45.312882+0000","last_clean":"2023-03-25T04:27:45.312882+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:27:45.312882+0000","last_undegraded":"2023-03-25T04:27:45.312882+0000","last_fullsized":"2023-03-25T04:27:45.312882+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.311730+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:27:45.311730+0000","last_peered":"2023-03-25T04:27:45.311730+0000","last_clean":"2023-03-25T04:27:45.311730+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:27:45.311730+0000","last_undegraded":"2023-03-25T04:27:45.311730+0000","last_fullsized":"2023-03-25T04:27:45.311730+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315735+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:27:45.315735+0000","last_peered":"2023-03-25T04:27:45.315735+0000","last_clean":"2023-03-25T04:27:45.315735+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:27:45.315735+0000","last_undegraded":"2023-03-25T04:27:45.315735+0000","last_fullsized":"2023-03-25T04:27:45.315735+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312833+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:27:45.312833+0000","last_peered":"2023-03-25T04:27:45.312833+0000","last_clean":"2023-03-25T04:27:45.312833+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:27:45.312833+0000","last_undegraded":"2023-03-25T04:27:45.312833+0000","last_fullsized":"2023-03-25T04:27:45.312833+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117040,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:30:37 2023","interfaces":[{"interface":"back","average":{"1min":0.28799999999999998,"5min":0.28899999999999998,"15min":0.29499999999999998},"min":{"1min":0.20200000000000001,"5min":0.191,"15min":0.191},"max":{"1min":0.433,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.29799999999999999},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.28999999999999998,"15min":0.29799999999999999},"min":{"1min":0.17499999999999999,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":0.40300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.25700000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051630,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:30:32 2023","interfaces":[{"interface":"back","average":{"1min":0.25600000000000001,"5min":0.255,"15min":0.251},"min":{"1min":0.19400000000000001,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.35699999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"last":0.22800000000000001},{"interface":"front","average":{"1min":0.28699999999999998,"5min":0.27200000000000002,"15min":0.26600000000000001},"min":{"1min":0.20599999999999999,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.39400000000000002,"5min":0.42899999999999999,"15min":0.42899999999999999},"last":0.24099999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:31:08.343 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace: 2023-03-25T04:31:24.283+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:31:25.192+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:31:25.295+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)

Ignoring malparsed backtrace: 2023-03-25T04:31:40.257 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:31:40.271 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":212,"stamp":"2023-03-25T04:31:40.144732+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002208"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315468+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:27:45.315468+0000","last_peered":"2023-03-25T04:27:45.315468+0000","last_clean":"2023-03-25T04:27:45.315468+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:27:45.315468+0000","last_undegraded":"2023-03-25T04:27:45.315468+0000","last_fullsized":"2023-03-25T04:27:45.315468+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312845+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:27:45.312845+0000","last_peered":"2023-03-25T04:27:45.312845+0000","last_clean":"2023-03-25T04:27:45.312845+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:27:45.312845+0000","last_undegraded":"2023-03-25T04:27:45.312845+0000","last_fullsized":"2023-03-25T04:27:45.312845+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:23.308607+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:29:23.308607+0000","last_peered":"2023-03-25T04:29:23.308607+0000","last_clean":"2023-03-25T04:29:23.308607+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:29:23.308607+0000","last_undegraded":"2023-03-25T04:29:23.308607+0000","last_fullsized":"2023-03-25T04:29:23.308607+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":28,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:20.554710+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:29:20.554710+0000","last_peered":"2023-03-25T04:29:20.554710+0000","last_clean":"2023-03-25T04:29:20.554710+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:29:20.554710+0000","last_undegraded":"2023-03-25T04:29:20.554710+0000","last_fullsized":"2023-03-25T04:29:20.554710+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309216+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:29:28.309216+0000","last_peered":"2023-03-25T04:29:28.309216+0000","last_clean":"2023-03-25T04:29:28.309216+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:29:28.309216+0000","last_undegraded":"2023-03-25T04:29:28.309216+0000","last_fullsized":"2023-03-25T04:29:28.309216+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309233+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:29:28.309233+0000","last_peered":"2023-03-25T04:29:28.309233+0000","last_clean":"2023-03-25T04:29:28.309233+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:29:28.309233+0000","last_undegraded":"2023-03-25T04:29:28.309233+0000","last_fullsized":"2023-03-25T04:29:28.309233+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117046,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:30:37 2023","interfaces":[{"interface":"back","average":{"1min":0.28799999999999998,"5min":0.28899999999999998,"15min":0.29499999999999998},"min":{"1min":0.20200000000000001,"5min":0.191,"15min":0.191},"max":{"1min":0.433,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.216},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.28999999999999998,"15min":0.29799999999999999},"min":{"1min":0.17499999999999999,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":0.40300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.24299999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051637,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:31:35 2023","interfaces":[{"interface":"back","average":{"1min":0.26100000000000001,"5min":0.25700000000000001,"15min":0.251},"min":{"1min":0.19,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.35899999999999999,"5min":0.41599999999999998,"15min":0.41599999999999998},"last":0.221},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.27400000000000002,"15min":0.26700000000000002},"min":{"1min":0.20200000000000001,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.374,"5min":0.42899999999999999,"15min":0.42899999999999999},"last":0.24099999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:32:01.906+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:32:02.388+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:32:02.794 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:32:22.326 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":233,"stamp":"2023-03-25T04:32:22.153289+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002545"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:33.309748+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:29:33.309748+0000","last_peered":"2023-03-25T04:29:33.309748+0000","last_clean":"2023-03-25T04:29:33.309748+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:29:33.309748+0000","last_undegraded":"2023-03-25T04:29:33.309748+0000","last_fullsized":"2023-03-25T04:29:33.309748+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:33.309729+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:29:33.309729+0000","last_peered":"2023-03-25T04:29:33.309729+0000","last_clean":"2023-03-25T04:29:33.309729+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:29:33.309729+0000","last_undegraded":"2023-03-25T04:29:33.309729+0000","last_fullsized":"2023-03-25T04:29:33.309729+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:23.308607+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:29:23.308607+0000","last_peered":"2023-03-25T04:29:23.308607+0000","last_clean":"2023-03-25T04:29:23.308607+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:29:23.308607+0000","last_undegraded":"2023-03-25T04:29:23.308607+0000","last_fullsized":"2023-03-25T04:29:23.308607+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":28,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:20.554710+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:29:20.554710+0000","last_peered":"2023-03-25T04:29:20.554710+0000","last_clean":"2023-03-25T04:29:20.554710+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:29:20.554710+0000","last_undegraded":"2023-03-25T04:29:20.554710+0000","last_fullsized":"2023-03-25T04:29:20.554710+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309216+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:29:28.309216+0000","last_peered":"2023-03-25T04:29:28.309216+0000","last_clean":"2023-03-25T04:29:28.309216+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:29:28.309216+0000","last_undegraded":"2023-03-25T04:29:28.309216+0000","last_fullsized":"2023-03-25T04:29:28.309216+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309233+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:29:28.309233+0000","last_peered":"2023-03-25T04:29:28.309233+0000","last_clean":"2023-03-25T04:29:28.309233+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:29:28.309233+0000","last_undegraded":"2023-03-25T04:29:28.309233+0000","last_fullsized":"2023-03-25T04:29:28.309233+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117055,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:31:42 2023","interfaces":[{"interface":"back","average":{"1min":0.27500000000000002,"5min":0.28399999999999997,"15min":0.29399999999999998},"min":{"1min":0.19800000000000001,"5min":0.191,"15min":0.191},"max":{"1min":0.36899999999999999,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.32300000000000001},{"interface":"front","average":{"1min":0.27900000000000003,"5min":0.28499999999999998,"15min":0.29599999999999999},"min":{"1min":0.216,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":0.40799999999999997,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.27100000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051646,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:31:35 2023","interfaces":[{"interface":"back","average":{"1min":0.26100000000000001,"5min":0.25700000000000001,"15min":0.251},"min":{"1min":0.19,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.35899999999999999,"5min":0.41599999999999998,"15min":0.41599999999999998},"last":0.25600000000000001},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.27400000000000002,"15min":0.26700000000000002},"min":{"1min":0.20200000000000001,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.374,"5min":0.42899999999999999,"15min":0.42899999999999999},"last":0.29199999999999998}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:32:22.327 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean
, 2023-03-25T04:32:22.327 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}

Ignoring malparsed backtrace: 2023-03-25T04:32:39.836 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:32:39.849 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":241,"stamp":"2023-03-25T04:32:38.156312+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002346"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:33.309748+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:29:33.309748+0000","last_peered":"2023-03-25T04:29:33.309748+0000","last_clean":"2023-03-25T04:29:33.309748+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:29:33.309748+0000","last_undegraded":"2023-03-25T04:29:33.309748+0000","last_fullsized":"2023-03-25T04:29:33.309748+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:33.309729+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:29:33.309729+0000","last_peered":"2023-03-25T04:29:33.309729+0000","last_clean":"2023-03-25T04:29:33.309729+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:29:33.309729+0000","last_undegraded":"2023-03-25T04:29:33.309729+0000","last_fullsized":"2023-03-25T04:29:33.309729+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:23.308607+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:29:23.308607+0000","last_peered":"2023-03-25T04:29:23.308607+0000","last_clean":"2023-03-25T04:29:23.308607+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:29:23.308607+0000","last_undegraded":"2023-03-25T04:29:23.308607+0000","last_fullsized":"2023-03-25T04:29:23.308607+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":28,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:20.554710+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:29:20.554710+0000","last_peered":"2023-03-25T04:29:20.554710+0000","last_clean":"2023-03-25T04:29:20.554710+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:29:20.554710+0000","last_undegraded":"2023-03-25T04:29:20.554710+0000","last_fullsized":"2023-03-25T04:29:20.554710+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309216+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:29:28.309216+0000","last_peered":"2023-03-25T04:29:28.309216+0000","last_clean":"2023-03-25T04:29:28.309216+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:29:28.309216+0000","last_undegraded":"2023-03-25T04:29:28.309216+0000","last_fullsized":"2023-03-25T04:29:28.309216+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309233+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:29:28.309233+0000","last_peered":"2023-03-25T04:29:28.309233+0000","last_clean":"2023-03-25T04:29:28.309233+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:29:28.309233+0000","last_undegraded":"2023-03-25T04:29:28.309233+0000","last_fullsized":"2023-03-25T04:29:28.309233+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117058,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:31:42 2023","interfaces":[{"interface":"back","average":{"1min":0.27500000000000002,"5min":0.28399999999999997,"15min":0.29399999999999998},"min":{"1min":0.19800000000000001,"5min":0.191,"15min":0.191},"max":{"1min":0.36899999999999999,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.30599999999999999},{"interface":"front","average":{"1min":0.27900000000000003,"5min":0.28499999999999998,"15min":0.29599999999999999},"min":{"1min":0.216,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":0.40799999999999997,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.32500000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051649,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:31:35 2023","interfaces":[{"interface":"back","average":{"1min":0.26100000000000001,"5min":0.25700000000000001,"15min":0.251},"min":{"1min":0.19,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.35899999999999999,"5min":0.41599999999999998,"15min":0.41599999999999998},"last":0.24199999999999999},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.27400000000000002,"15min":0.26700000000000002},"min":{"1min":0.20200000000000001,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.374,"5min":0.42899999999999999,"15min":0.42899999999999999},"last":0.22600000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:32:54.944+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:32:55.469+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:32:55.927+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)

Ignoring malparsed backtrace: 2023-03-25T04:33:13.958 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean
, 2023-03-25T04:33:13.959 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}
, 2023-03-25T04:33:13.959 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json

Ignoring malparsed backtrace: 2023-03-25T04:33:31.199 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":267,"stamp":"2023-03-25T04:33:30.166660+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002269"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:33.309748+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:29:33.309748+0000","last_peered":"2023-03-25T04:29:33.309748+0000","last_clean":"2023-03-25T04:29:33.309748+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:29:33.309748+0000","last_undegraded":"2023-03-25T04:29:33.309748+0000","last_fullsized":"2023-03-25T04:29:33.309748+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:33.309729+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:29:33.309729+0000","last_peered":"2023-03-25T04:29:33.309729+0000","last_clean":"2023-03-25T04:29:33.309729+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:29:33.309729+0000","last_undegraded":"2023-03-25T04:29:33.309729+0000","last_fullsized":"2023-03-25T04:29:33.309729+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":28,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:31:23.320894+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:31:23.320894+0000","last_peered":"2023-03-25T04:31:23.320894+0000","last_clean":"2023-03-25T04:31:23.320894+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:31:23.320894+0000","last_undegraded":"2023-03-25T04:31:23.320894+0000","last_fullsized":"2023-03-25T04:31:23.320894+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":28,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:20.554710+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:29:20.554710+0000","last_peered":"2023-03-25T04:29:20.554710+0000","last_clean":"2023-03-25T04:29:20.554710+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:29:20.554710+0000","last_undegraded":"2023-03-25T04:29:20.554710+0000","last_fullsized":"2023-03-25T04:29:20.554710+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309216+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:29:28.309216+0000","last_peered":"2023-03-25T04:29:28.309216+0000","last_clean":"2023-03-25T04:29:28.309216+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:29:28.309216+0000","last_undegraded":"2023-03-25T04:29:28.309216+0000","last_fullsized":"2023-03-25T04:29:28.309216+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309233+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:29:28.309233+0000","last_peered":"2023-03-25T04:29:28.309233+0000","last_clean":"2023-03-25T04:29:28.309233+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:29:28.309233+0000","last_undegraded":"2023-03-25T04:29:28.309233+0000","last_fullsized":"2023-03-25T04:29:28.309233+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117068,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:32:46 2023","interfaces":[{"interface":"back","average":{"1min":0.27000000000000002,"5min":0.27900000000000003,"15min":0.29199999999999998},"min":{"1min":0.17899999999999999,"5min":0.17899999999999999,"15min":0.17899999999999999},"max":{"1min":0.35599999999999998,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.28799999999999998},{"interface":"front","average":{"1min":0.26800000000000002,"5min":0.27800000000000002,"15min":0.29399999999999998},"min":{"1min":0.19900000000000001,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":0.377,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.315}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051660,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:32:36 2023","interfaces":[{"interface":"back","average":{"1min":0.248,"5min":0.25700000000000001,"15min":0.251},"min":{"1min":0.17000000000000001,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"last":0.22700000000000001},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.27600000000000002,"15min":0.26800000000000002},"min":{"1min":0.192,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"last":0.20399999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:33:31.201 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:33:31.400+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)

Ignoring malparsed backtrace: 2023-03-25T04:33:47.531+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:33:47.951+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:33:48.505+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)

Ignoring malparsed backtrace: 2023-03-25T04:34:06.104 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json
, 2023-03-25T04:34:06.118+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:34:06.497 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:34:23.399 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:34:23.665+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:34:23.712 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:34:40.195+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:34:40.523+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:34:40.676 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace: 2023-03-25T04:35:03.381+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:35:04.139+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:35:04.391+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)

Ignoring malparsed backtrace: 2023-03-25T04:35:22.738 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean
, 2023-03-25T04:35:22.739 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}
, 2023-03-25T04:35:22.739 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json

Ignoring malparsed backtrace: 2023-03-25T04:35:41.393+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:35:42.352+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:35:42.376+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:35:57.384 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":340,"stamp":"2023-03-25T04:35:56.194992+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.001930"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":36,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:31:43.322894+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:31:43.322894+0000","last_peered":"2023-03-25T04:31:43.322894+0000","last_clean":"2023-03-25T04:31:43.322894+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:31:43.322894+0000","last_undegraded":"2023-03-25T04:31:43.322894+0000","last_fullsized":"2023-03-25T04:31:43.322894+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":28,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:31:43.322876+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:31:43.322876+0000","last_peered":"2023-03-25T04:31:43.322876+0000","last_clean":"2023-03-25T04:31:43.322876+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:31:43.322876+0000","last_undegraded":"2023-03-25T04:31:43.322876+0000","last_fullsized":"2023-03-25T04:31:43.322876+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:23.331998+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:33:23.331998+0000","last_peered":"2023-03-25T04:33:23.331998+0000","last_clean":"2023-03-25T04:33:23.331998+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:33:23.331998+0000","last_undegraded":"2023-03-25T04:33:23.331998+0000","last_fullsized":"2023-03-25T04:33:23.331998+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:30.576723+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:33:30.576723+0000","last_peered":"2023-03-25T04:33:30.576723+0000","last_clean":"2023-03-25T04:33:30.576723+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:33:30.576723+0000","last_undegraded":"2023-03-25T04:33:30.576723+0000","last_fullsized":"2023-03-25T04:33:30.576723+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":37,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:38.333447+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:33:38.333447+0000","last_peered":"2023-03-25T04:33:38.333447+0000","last_clean":"2023-03-25T04:33:38.333447+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:33:38.333447+0000","last_undegraded":"2023-03-25T04:33:38.333447+0000","last_fullsized":"2023-03-25T04:33:38.333447+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:38.333467+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:33:38.333467+0000","last_peered":"2023-03-25T04:33:38.333467+0000","last_clean":"2023-03-25T04:33:38.333467+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:33:38.333467+0000","last_undegraded":"2023-03-25T04:33:38.333467+0000","last_fullsized":"2023-03-25T04:33:38.333467+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117098,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:35:49 2023","interfaces":[{"interface":"back","average":{"1min":0.255,"5min":0.26000000000000001,"15min":0.28199999999999997},"min":{"1min":0.17899999999999999,"5min":0.16,"15min":0.16},"max":{"1min":0.36399999999999999,"5min":0.42599999999999999,"15min":0.45100000000000001},"last":0.26100000000000001},{"interface":"front","average":{"1min":0.26000000000000001,"5min":0.26700000000000002,"15min":0.28599999999999998},"min":{"1min":0.188,"5min":0.17999999999999999,"15min":0.17499999999999999},"max":{"1min":0.39100000000000001,"5min":0.45200000000000001,"15min":0.46300000000000002},"last":0.28399999999999997}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051689,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:35:42 2023","interfaces":[{"interface":"back","average":{"1min":0.29399999999999998,"5min":0.27200000000000002,"15min":0.25800000000000001},"min":{"1min":0.17299999999999999,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.41099999999999998,"5min":0.42699999999999999,"15min":0.42699999999999999},"last":0.20399999999999999},{"interface":"front","average":{"1min":0.30299999999999999,"5min":0.28399999999999997,"15min":0.27300000000000002},"min":{"1min":0.19700000000000001,"5min":0.192,"15min":0.16600000000000001},"max":{"1min":0.43099999999999999,"5min":0.45800000000000002,"15min":0.45800000000000002},"last":0.22}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:35:57.624+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)

Ignoring malparsed backtrace: 2023-03-25T04:36:17.226 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:36:17.239 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":350,"stamp":"2023-03-25T04:36:16.198866+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002148"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":37,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:53.335026+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:33:53.335026+0000","last_peered":"2023-03-25T04:33:53.335026+0000","last_clean":"2023-03-25T04:33:53.335026+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:33:53.335026+0000","last_undegraded":"2023-03-25T04:33:53.335026+0000","last_fullsized":"2023-03-25T04:33:53.335026+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:53.335009+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:33:53.335009+0000","last_peered":"2023-03-25T04:33:53.335009+0000","last_clean":"2023-03-25T04:33:53.335009+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:33:53.335009+0000","last_undegraded":"2023-03-25T04:33:53.335009+0000","last_fullsized":"2023-03-25T04:33:53.335009+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:23.331998+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:33:23.331998+0000","last_peered":"2023-03-25T04:33:23.331998+0000","last_clean":"2023-03-25T04:33:23.331998+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:33:23.331998+0000","last_undegraded":"2023-03-25T04:33:23.331998+0000","last_fullsized":"2023-03-25T04:33:23.331998+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:30.576723+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:33:30.576723+0000","last_peered":"2023-03-25T04:33:30.576723+0000","last_clean":"2023-03-25T04:33:30.576723+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:33:30.576723+0000","last_undegraded":"2023-03-25T04:33:30.576723+0000","last_fullsized":"2023-03-25T04:33:30.576723+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":37,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:38.333447+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:33:38.333447+0000","last_peered":"2023-03-25T04:33:38.333447+0000","last_clean":"2023-03-25T04:33:38.333447+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:33:38.333447+0000","last_undegraded":"2023-03-25T04:33:38.333447+0000","last_fullsized":"2023-03-25T04:33:38.333447+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:38.333467+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:33:38.333467+0000","last_peered":"2023-03-25T04:33:38.333467+0000","last_clean":"2023-03-25T04:33:38.333467+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:33:38.333467+0000","last_undegraded":"2023-03-25T04:33:38.333467+0000","last_fullsized":"2023-03-25T04:33:38.333467+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117102,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:35:49 2023","interfaces":[{"interface":"back","average":{"1min":0.255,"5min":0.26000000000000001,"15min":0.28199999999999997},"min":{"1min":0.17899999999999999,"5min":0.16,"15min":0.16},"max":{"1min":0.36399999999999999,"5min":0.42599999999999999,"15min":0.45100000000000001},"last":0.28399999999999997},{"interface":"front","average":{"1min":0.26000000000000001,"5min":0.26700000000000002,"15min":0.28599999999999998},"min":{"1min":0.188,"5min":0.17999999999999999,"15min":0.17499999999999999},"max":{"1min":0.39100000000000001,"5min":0.45200000000000001,"15min":0.46300000000000002},"last":0.30299999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051693,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:35:42 2023","interfaces":[{"interface":"back","average":{"1min":0.29399999999999998,"5min":0.27200000000000002,"15min":0.25800000000000001},"min":{"1min":0.17299999999999999,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.41099999999999998,"5min":0.42699999999999999,"15min":0.42699999999999999},"last":0.35099999999999998},{"interface":"front","average":{"1min":0.30299999999999999,"5min":0.28399999999999997,"15min":0.27300000000000002},"min":{"1min":0.19700000000000001,"5min":0.192,"15min":0.16600000000000001},"max":{"1min":0.43099999999999999,"5min":0.45800000000000002,"15min":0.45800000000000002},"last":0.32900000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:36:34.556+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:36:34.717+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:36:35.583+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)

Ignoring malparsed backtrace: 2023-03-25T04:36:52.680 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json
, 2023-03-25T04:36:52.701+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:36:53.067 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:37:09.963 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:37:10.273 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all

Ignoring malparsed backtrace: 2023-03-25T04:37:26.856+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:37:27.288 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:37:27.538+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:37:41.903 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":392,"stamp":"2023-03-25T04:37:40.213373+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002012"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":37,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:53.335026+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:33:53.335026+0000","last_peered":"2023-03-25T04:33:53.335026+0000","last_clean":"2023-03-25T04:33:53.335026+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:33:53.335026+0000","last_undegraded":"2023-03-25T04:33:53.335026+0000","last_fullsized":"2023-03-25T04:33:53.335026+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:53.335009+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:33:53.335009+0000","last_peered":"2023-03-25T04:33:53.335009+0000","last_clean":"2023-03-25T04:33:53.335009+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:33:53.335009+0000","last_undegraded":"2023-03-25T04:33:53.335009+0000","last_fullsized":"2023-03-25T04:33:53.335009+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:23.343585+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:35:23.343585+0000","last_peered":"2023-03-25T04:35:23.343585+0000","last_clean":"2023-03-25T04:35:23.343585+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:35:23.343585+0000","last_undegraded":"2023-03-25T04:35:23.343585+0000","last_fullsized":"2023-03-25T04:35:23.343585+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:30.576723+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:33:30.576723+0000","last_peered":"2023-03-25T04:33:30.576723+0000","last_clean":"2023-03-25T04:33:30.576723+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:33:30.576723+0000","last_undegraded":"2023-03-25T04:33:30.576723+0000","last_fullsized":"2023-03-25T04:33:30.576723+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":37,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:38.333447+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:33:38.333447+0000","last_peered":"2023-03-25T04:33:38.333447+0000","last_clean":"2023-03-25T04:33:38.333447+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:33:38.333447+0000","last_undegraded":"2023-03-25T04:33:38.333447+0000","last_fullsized":"2023-03-25T04:33:38.333447+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:38.333467+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:33:38.333467+0000","last_peered":"2023-03-25T04:33:38.333467+0000","last_clean":"2023-03-25T04:33:38.333467+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:33:38.333467+0000","last_undegraded":"2023-03-25T04:33:38.333467+0000","last_fullsized":"2023-03-25T04:33:38.333467+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117119,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:36:50 2023","interfaces":[{"interface":"back","average":{"1min":0.254,"5min":0.25600000000000001,"15min":0.27900000000000003},"min":{"1min":0.20499999999999999,"5min":0.16,"15min":0.16},"max":{"1min":0.371,"5min":0.42599999999999999,"15min":0.45100000000000001},"last":0.23000000000000001},{"interface":"front","average":{"1min":0.26800000000000002,"5min":0.26500000000000001,"15min":0.28399999999999997},"min":{"1min":0.19700000000000001,"5min":0.17999999999999999,"15min":0.17499999999999999},"max":{"1min":0.34699999999999998,"5min":0.45200000000000001,"15min":0.46300000000000002},"last":0.20899999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051710,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:36:44 2023","interfaces":[{"interface":"back","average":{"1min":0.28399999999999997,"5min":0.27700000000000002,"15min":0.26100000000000001},"min":{"1min":0.192,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.40899999999999997,"5min":0.42699999999999999,"15min":0.42699999999999999},"last":0.39700000000000002},{"interface":"front","average":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.27500000000000002},"min":{"1min":0.182,"5min":0.182,"15min":0.16600000000000001},"max":{"1min":0.439,"5min":0.45800000000000002,"15min":0.45800000000000002},"last":0.439}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:37:42.400+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)

Ignoring malparsed backtrace: 2023-03-25T04:38:04.766+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:38:04.922 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:38:05.246 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:38:19.605 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":411,"stamp":"2023-03-25T04:38:18.220966+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002368"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":38,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:36:03.347070+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:36:03.347070+0000","last_peered":"2023-03-25T04:36:03.347070+0000","last_clean":"2023-03-25T04:36:03.347070+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:36:03.347070+0000","last_undegraded":"2023-03-25T04:36:03.347070+0000","last_fullsized":"2023-03-25T04:36:03.347070+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:36:03.347025+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:36:03.347025+0000","last_peered":"2023-03-25T04:36:03.347025+0000","last_clean":"2023-03-25T04:36:03.347025+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:36:03.347025+0000","last_undegraded":"2023-03-25T04:36:03.347025+0000","last_fullsized":"2023-03-25T04:36:03.347025+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:23.343585+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:35:23.343585+0000","last_peered":"2023-03-25T04:35:23.343585+0000","last_clean":"2023-03-25T04:35:23.343585+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:35:23.343585+0000","last_undegraded":"2023-03-25T04:35:23.343585+0000","last_fullsized":"2023-03-25T04:35:23.343585+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:35.587025+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:35:35.587025+0000","last_peered":"2023-03-25T04:35:35.587025+0000","last_clean":"2023-03-25T04:35:35.587025+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:35:35.587025+0000","last_undegraded":"2023-03-25T04:35:35.587025+0000","last_fullsized":"2023-03-25T04:35:35.587025+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":38,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:43.345308+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:35:43.345308+0000","last_peered":"2023-03-25T04:35:43.345308+0000","last_clean":"2023-03-25T04:35:43.345308+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:35:43.345308+0000","last_undegraded":"2023-03-25T04:35:43.345308+0000","last_fullsized":"2023-03-25T04:35:43.345308+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:43.345324+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:35:43.345324+0000","last_peered":"2023-03-25T04:35:43.345324+0000","last_clean":"2023-03-25T04:35:43.345324+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:35:43.345324+0000","last_undegraded":"2023-03-25T04:35:43.345324+0000","last_fullsized":"2023-03-25T04:35:43.345324+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117127,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:37:53 2023","interfaces":[{"interface":"back","average":{"1min":0.25800000000000001,"5min":0.253,"15min":0.27700000000000002},"min":{"1min":0.184,"5min":0.16,"15min":0.16},"max":{"1min":0.44600000000000001,"5min":0.44600000000000001,"15min":0.45100000000000001},"last":0.35199999999999998},{"interface":"front","average":{"1min":0.26900000000000002,"5min":0.26500000000000001,"15min":0.28199999999999997},"min":{"1min":0.192,"5min":0.17999999999999999,"15min":0.17499999999999999},"max":{"1min":0.46400000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"last":0.33200000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051717,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:37:44 2023","interfaces":[{"interface":"back","average":{"1min":0.27300000000000002,"5min":0.28199999999999997,"15min":0.26200000000000001},"min":{"1min":0.17899999999999999,"5min":0.17299999999999999,"15min":0.17000000000000001},"max":{"1min":0.41799999999999998,"5min":0.42099999999999999,"15min":0.42699999999999999},"last":0.28699999999999998},{"interface":"front","average":{"1min":0.29399999999999998,"5min":0.29099999999999998,"15min":0.27700000000000002},"min":{"1min":0.19900000000000001,"5min":0.182,"15min":0.16600000000000001},"max":{"1min":0.441,"5min":0.441,"15min":0.45800000000000002},"last":0.26300000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:38:19.718+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:38:20.315+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)

Ignoring malparsed backtrace: 2023-03-25T04:38:44.283 INFO:teuthology.orchestra.run.smithi070.stdout:
, 2023-03-25T04:38:44.283 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":781,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":16,"modified":"2023-03-25T04:38:40.225321+0000","services":{}},"progress_events":{}}
, 2023-03-25T04:38:44.296 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now

Ignoring malparsed backtrace: 2023-03-25T04:39:02.477+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:39:02.578+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:39:03.524+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:39:20.359 INFO:teuthology.orchestra.run.smithi070.stdout:
, 2023-03-25T04:39:20.359 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":818,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":16,"modified":"2023-03-25T04:38:40.225321+0000","services":{}},"progress_events":{}}
, 2023-03-25T04:39:20.372 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now

Ignoring malparsed backtrace: 2023-03-25T04:39:37.527 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:39:37.541 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":450,"stamp":"2023-03-25T04:39:36.235332+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002086"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":38,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:36:03.347070+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:36:03.347070+0000","last_peered":"2023-03-25T04:36:03.347070+0000","last_clean":"2023-03-25T04:36:03.347070+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:36:03.347070+0000","last_undegraded":"2023-03-25T04:36:03.347070+0000","last_fullsized":"2023-03-25T04:36:03.347070+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:36:03.347025+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:36:03.347025+0000","last_peered":"2023-03-25T04:36:03.347025+0000","last_clean":"2023-03-25T04:36:03.347025+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:36:03.347025+0000","last_undegraded":"2023-03-25T04:36:03.347025+0000","last_fullsized":"2023-03-25T04:36:03.347025+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:23.353778+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:37:23.353778+0000","last_peered":"2023-03-25T04:37:23.353778+0000","last_clean":"2023-03-25T04:37:23.353778+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:37:23.353778+0000","last_undegraded":"2023-03-25T04:37:23.353778+0000","last_fullsized":"2023-03-25T04:37:23.353778+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:35.587025+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:35:35.587025+0000","last_peered":"2023-03-25T04:35:35.587025+0000","last_clean":"2023-03-25T04:35:35.587025+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:35:35.587025+0000","last_undegraded":"2023-03-25T04:35:35.587025+0000","last_fullsized":"2023-03-25T04:35:35.587025+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":38,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:43.345308+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:35:43.345308+0000","last_peered":"2023-03-25T04:35:43.345308+0000","last_clean":"2023-03-25T04:35:43.345308+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:35:43.345308+0000","last_undegraded":"2023-03-25T04:35:43.345308+0000","last_fullsized":"2023-03-25T04:35:43.345308+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:43.345324+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:35:43.345324+0000","last_peered":"2023-03-25T04:35:43.345324+0000","last_clean":"2023-03-25T04:35:43.345324+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:35:43.345324+0000","last_undegraded":"2023-03-25T04:35:43.345324+0000","last_fullsized":"2023-03-25T04:35:43.345324+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117143,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:38:54 2023","interfaces":[{"interface":"back","average":{"1min":0.25800000000000001,"5min":0.25700000000000001,"15min":0.27400000000000002},"min":{"1min":0.192,"5min":0.17399999999999999,"15min":0.16},"max":{"1min":0.39400000000000002,"5min":0.44600000000000001,"15min":0.45100000000000001},"last":0.26500000000000001},{"interface":"front","average":{"1min":0.25900000000000001,"5min":0.26700000000000002,"15min":0.27900000000000003},"min":{"1min":0.19500000000000001,"5min":0.188,"15min":0.17499999999999999},"max":{"1min":0.41499999999999998,"5min":0.46400000000000002,"15min":0.46400000000000002},"last":0.29099999999999998}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051733,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:38:45 2023","interfaces":[{"interface":"back","average":{"1min":0.27300000000000002,"5min":0.28100000000000003,"15min":0.26400000000000001},"min":{"1min":0.19400000000000001,"5min":0.17299999999999999,"15min":0.17000000000000001},"max":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"last":0.27300000000000002},{"interface":"front","average":{"1min":0.28799999999999998,"5min":0.29099999999999998,"15min":0.27800000000000002},"min":{"1min":0.19800000000000001,"5min":0.182,"15min":0.16600000000000001},"max":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"last":0.308}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:39:54.416+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:39:54.455 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:39:54.780 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:40:09.288+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:40:09.389+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:40:10.267+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)

Ignoring malparsed backtrace: 2023-03-25T04:40:28.984 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":476,"stamp":"2023-03-25T04:40:28.245911+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002693"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":39,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:38:13.358968+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:38:13.358968+0000","last_peered":"2023-03-25T04:38:13.358968+0000","last_clean":"2023-03-25T04:38:13.358968+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:38:13.358968+0000","last_undegraded":"2023-03-25T04:38:13.358968+0000","last_fullsized":"2023-03-25T04:38:13.358968+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:38:13.358922+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:38:13.358922+0000","last_peered":"2023-03-25T04:38:13.358922+0000","last_clean":"2023-03-25T04:38:13.358922+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:38:13.358922+0000","last_undegraded":"2023-03-25T04:38:13.358922+0000","last_fullsized":"2023-03-25T04:38:13.358922+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:23.353778+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:37:23.353778+0000","last_peered":"2023-03-25T04:37:23.353778+0000","last_clean":"2023-03-25T04:37:23.353778+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:37:23.353778+0000","last_undegraded":"2023-03-25T04:37:23.353778+0000","last_fullsized":"2023-03-25T04:37:23.353778+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:40.598579+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:37:40.598579+0000","last_peered":"2023-03-25T04:37:40.598579+0000","last_clean":"2023-03-25T04:37:40.598579+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:37:40.598579+0000","last_undegraded":"2023-03-25T04:37:40.598579+0000","last_fullsized":"2023-03-25T04:37:40.598579+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":39,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:48.356344+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:37:48.356344+0000","last_peered":"2023-03-25T04:37:48.356344+0000","last_clean":"2023-03-25T04:37:48.356344+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:37:48.356344+0000","last_undegraded":"2023-03-25T04:37:48.356344+0000","last_fullsized":"2023-03-25T04:37:48.356344+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:48.356365+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:37:48.356365+0000","last_peered":"2023-03-25T04:37:48.356365+0000","last_clean":"2023-03-25T04:37:48.356365+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:37:48.356365+0000","last_undegraded":"2023-03-25T04:37:48.356365+0000","last_fullsized":"2023-03-25T04:37:48.356365+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117153,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:39:54 2023","interfaces":[{"interface":"back","average":{"1min":0.25900000000000001,"5min":0.25600000000000001,"15min":0.27100000000000002},"min":{"1min":0.188,"5min":0.17899999999999999,"15min":0.16},"max":{"1min":0.41999999999999998,"5min":0.44600000000000001,"15min":0.45100000000000001},"last":0.255},{"interface":"front","average":{"1min":0.27500000000000002,"5min":0.26600000000000001,"15min":0.27700000000000002},"min":{"1min":0.17699999999999999,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.45600000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"last":0.26800000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051743,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:39:47 2023","interfaces":[{"interface":"back","average":{"1min":0.26300000000000001,"5min":0.27700000000000002,"15min":0.26500000000000001},"min":{"1min":0.187,"5min":0.17299999999999999,"15min":0.17000000000000001},"max":{"1min":0.433,"5min":0.433,"15min":0.433},"last":0.36599999999999999},{"interface":"front","average":{"1min":0.27100000000000002,"5min":0.28799999999999998,"15min":0.27900000000000003},"min":{"1min":0.20100000000000001,"5min":0.182,"15min":0.16600000000000001},"max":{"1min":0.48599999999999999,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.34999999999999998}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:40:28.986 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean
, 2023-03-25T04:40:28.986 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}

Ignoring malparsed backtrace: 2023-03-25T04:40:47.229+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:40:47.503+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:40:48.251+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)

Ignoring malparsed backtrace: 2023-03-25T04:41:03.267 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:41:03.578 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all

Ignoring malparsed backtrace: 2023-03-25T04:41:21.347 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":502,"stamp":"2023-03-25T04:41:20.256204+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002342"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":39,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:38:13.358968+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:38:13.358968+0000","last_peered":"2023-03-25T04:38:13.358968+0000","last_clean":"2023-03-25T04:38:13.358968+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:38:13.358968+0000","last_undegraded":"2023-03-25T04:38:13.358968+0000","last_fullsized":"2023-03-25T04:38:13.358968+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:38:13.358922+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:38:13.358922+0000","last_peered":"2023-03-25T04:38:13.358922+0000","last_clean":"2023-03-25T04:38:13.358922+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:38:13.358922+0000","last_undegraded":"2023-03-25T04:38:13.358922+0000","last_fullsized":"2023-03-25T04:38:13.358922+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:23.353778+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:37:23.353778+0000","last_peered":"2023-03-25T04:37:23.353778+0000","last_clean":"2023-03-25T04:37:23.353778+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:37:23.353778+0000","last_undegraded":"2023-03-25T04:37:23.353778+0000","last_fullsized":"2023-03-25T04:37:23.353778+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:40.598579+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:37:40.598579+0000","last_peered":"2023-03-25T04:37:40.598579+0000","last_clean":"2023-03-25T04:37:40.598579+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:37:40.598579+0000","last_undegraded":"2023-03-25T04:37:40.598579+0000","last_fullsized":"2023-03-25T04:37:40.598579+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":39,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:48.356344+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:37:48.356344+0000","last_peered":"2023-03-25T04:37:48.356344+0000","last_clean":"2023-03-25T04:37:48.356344+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:37:48.356344+0000","last_undegraded":"2023-03-25T04:37:48.356344+0000","last_fullsized":"2023-03-25T04:37:48.356344+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:48.356365+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:37:48.356365+0000","last_peered":"2023-03-25T04:37:48.356365+0000","last_clean":"2023-03-25T04:37:48.356365+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:37:48.356365+0000","last_undegraded":"2023-03-25T04:37:48.356365+0000","last_fullsized":"2023-03-25T04:37:48.356365+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117163,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:40:56 2023","interfaces":[{"interface":"back","average":{"1min":0.28799999999999998,"5min":0.26300000000000001,"15min":0.27100000000000002},"min":{"1min":0.188,"5min":0.184,"15min":0.16},"max":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.17499999999999999},{"interface":"front","average":{"1min":0.29599999999999999,"5min":0.27300000000000002,"15min":0.27700000000000002},"min":{"1min":0.20599999999999999,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.43099999999999999,"5min":0.46400000000000002,"15min":0.46400000000000002},"last":0.20699999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051754,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:40:52 2023","interfaces":[{"interface":"back","average":{"1min":0.28599999999999998,"5min":0.27500000000000002,"15min":0.26700000000000002},"min":{"1min":0.17899999999999999,"5min":0.17899999999999999,"15min":0.17000000000000001},"max":{"1min":0.44,"5min":0.44,"15min":0.44},"last":0.23699999999999999},{"interface":"front","average":{"1min":0.29799999999999999,"5min":0.28699999999999998,"15min":0.28100000000000003},"min":{"1min":0.20300000000000001,"5min":0.182,"15min":0.16600000000000001},"max":{"1min":0.47599999999999998,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.28000000000000003}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:41:21.349 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean
, 2023-03-25T04:41:21.349 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}

Ignoring malparsed backtrace: 2023-03-25T04:41:39.867+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:41:40.383+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:41:40.905+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:41:58.219 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json
, 2023-03-25T04:41:58.319+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:41:58.644 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:42:18.193 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":530,"stamp":"2023-03-25T04:42:16.266659+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002356"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":39,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:38:13.358968+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:38:13.358968+0000","last_peered":"2023-03-25T04:38:13.358968+0000","last_clean":"2023-03-25T04:38:13.358968+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:38:13.358968+0000","last_undegraded":"2023-03-25T04:38:13.358968+0000","last_fullsized":"2023-03-25T04:38:13.358968+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:38:13.358922+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:38:13.358922+0000","last_peered":"2023-03-25T04:38:13.358922+0000","last_clean":"2023-03-25T04:38:13.358922+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:38:13.358922+0000","last_undegraded":"2023-03-25T04:38:13.358922+0000","last_fullsized":"2023-03-25T04:38:13.358922+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:23.364593+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:39:23.364593+0000","last_peered":"2023-03-25T04:39:23.364593+0000","last_clean":"2023-03-25T04:39:23.364593+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:39:23.364593+0000","last_undegraded":"2023-03-25T04:39:23.364593+0000","last_fullsized":"2023-03-25T04:39:23.364593+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:45.608475+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:39:45.608475+0000","last_peered":"2023-03-25T04:39:45.608475+0000","last_clean":"2023-03-25T04:39:45.608475+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:39:45.608475+0000","last_undegraded":"2023-03-25T04:39:45.608475+0000","last_fullsized":"2023-03-25T04:39:45.608475+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":40,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:53.367398+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:39:53.367398+0000","last_peered":"2023-03-25T04:39:53.367398+0000","last_clean":"2023-03-25T04:39:53.367398+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:39:53.367398+0000","last_undegraded":"2023-03-25T04:39:53.367398+0000","last_fullsized":"2023-03-25T04:39:53.367398+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:53.367424+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:39:53.367424+0000","last_peered":"2023-03-25T04:39:53.367424+0000","last_clean":"2023-03-25T04:39:53.367424+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:39:53.367424+0000","last_undegraded":"2023-03-25T04:39:53.367424+0000","last_fullsized":"2023-03-25T04:39:53.367424+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117175,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:41:58 2023","interfaces":[{"interface":"back","average":{"1min":0.25800000000000001,"5min":0.26400000000000001,"15min":0.26800000000000002},"min":{"1min":0.17499999999999999,"5min":0.17499999999999999,"15min":0.16},"max":{"1min":0.39100000000000001,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.22900000000000001},{"interface":"front","average":{"1min":0.26300000000000001,"5min":0.27200000000000002,"15min":0.27400000000000002},"min":{"1min":0.20599999999999999,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.35699999999999998,"5min":0.46400000000000002,"15min":0.46400000000000002},"last":0.26400000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051765,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:41:52 2023","interfaces":[{"interface":"back","average":{"1min":0.26500000000000001,"5min":0.27200000000000002,"15min":0.26800000000000002},"min":{"1min":0.17000000000000001,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.39200000000000002,"5min":0.44,"15min":0.44},"last":0.23899999999999999},{"interface":"front","average":{"1min":0.28100000000000003,"5min":0.28599999999999998,"15min":0.28199999999999997},"min":{"1min":0.19400000000000001,"5min":0.19400000000000001,"15min":0.16600000000000001},"max":{"1min":0.41399999999999998,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.254}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:42:18.297+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:42:18.958+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:42:35.512 INFO:teuthology.orchestra.run.smithi070.stdout:
, 2023-03-25T04:42:35.512 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":1013,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":17,"modified":"2023-03-25T04:41:54.262744+0000","services":{}},"progress_events":{}}
, 2023-03-25T04:42:35.525 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:42:52.741 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":548,"stamp":"2023-03-25T04:42:52.273601+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002221"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":40,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:40:23.370056+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:40:23.370056+0000","last_peered":"2023-03-25T04:40:23.370056+0000","last_clean":"2023-03-25T04:40:23.370056+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:40:23.370056+0000","last_undegraded":"2023-03-25T04:40:23.370056+0000","last_fullsized":"2023-03-25T04:40:23.370056+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:40:23.370039+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:40:23.370039+0000","last_peered":"2023-03-25T04:40:23.370039+0000","last_clean":"2023-03-25T04:40:23.370039+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:40:23.370039+0000","last_undegraded":"2023-03-25T04:40:23.370039+0000","last_fullsized":"2023-03-25T04:40:23.370039+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:23.364593+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:39:23.364593+0000","last_peered":"2023-03-25T04:39:23.364593+0000","last_clean":"2023-03-25T04:39:23.364593+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:39:23.364593+0000","last_undegraded":"2023-03-25T04:39:23.364593+0000","last_fullsized":"2023-03-25T04:39:23.364593+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:45.608475+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:39:45.608475+0000","last_peered":"2023-03-25T04:39:45.608475+0000","last_clean":"2023-03-25T04:39:45.608475+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:39:45.608475+0000","last_undegraded":"2023-03-25T04:39:45.608475+0000","last_fullsized":"2023-03-25T04:39:45.608475+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":40,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:53.367398+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:39:53.367398+0000","last_peered":"2023-03-25T04:39:53.367398+0000","last_clean":"2023-03-25T04:39:53.367398+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:39:53.367398+0000","last_undegraded":"2023-03-25T04:39:53.367398+0000","last_fullsized":"2023-03-25T04:39:53.367398+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:53.367424+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:39:53.367424+0000","last_peered":"2023-03-25T04:39:53.367424+0000","last_clean":"2023-03-25T04:39:53.367424+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:39:53.367424+0000","last_undegraded":"2023-03-25T04:39:53.367424+0000","last_fullsized":"2023-03-25T04:39:53.367424+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117182,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:41:58 2023","interfaces":[{"interface":"back","average":{"1min":0.25800000000000001,"5min":0.26400000000000001,"15min":0.26800000000000002},"min":{"1min":0.17499999999999999,"5min":0.17499999999999999,"15min":0.16},"max":{"1min":0.39100000000000001,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.19600000000000001},{"interface":"front","average":{"1min":0.26300000000000001,"5min":0.27200000000000002,"15min":0.27400000000000002},"min":{"1min":0.20599999999999999,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.35699999999999998,"5min":0.46400000000000002,"15min":0.46400000000000002},"last":0.219}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051773,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:41:52 2023","interfaces":[{"interface":"back","average":{"1min":0.26500000000000001,"5min":0.27200000000000002,"15min":0.26800000000000002},"min":{"1min":0.17000000000000001,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.39200000000000002,"5min":0.44,"15min":0.44},"last":0.19},{"interface":"front","average":{"1min":0.28100000000000003,"5min":0.28599999999999998,"15min":0.28199999999999997},"min":{"1min":0.19400000000000001,"5min":0.19400000000000001,"15min":0.16600000000000001},"max":{"1min":0.41399999999999998,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.17599999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:42:52.742 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean

Ignoring malparsed backtrace: 2023-03-25T04:43:09.973 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:43:09.986 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":556,"stamp":"2023-03-25T04:43:08.276556+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002359"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":40,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:40:23.370056+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:40:23.370056+0000","last_peered":"2023-03-25T04:40:23.370056+0000","last_clean":"2023-03-25T04:40:23.370056+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:40:23.370056+0000","last_undegraded":"2023-03-25T04:40:23.370056+0000","last_fullsized":"2023-03-25T04:40:23.370056+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:40:23.370039+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:40:23.370039+0000","last_peered":"2023-03-25T04:40:23.370039+0000","last_clean":"2023-03-25T04:40:23.370039+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:40:23.370039+0000","last_undegraded":"2023-03-25T04:40:23.370039+0000","last_fullsized":"2023-03-25T04:40:23.370039+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:23.364593+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:39:23.364593+0000","last_peered":"2023-03-25T04:39:23.364593+0000","last_clean":"2023-03-25T04:39:23.364593+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:39:23.364593+0000","last_undegraded":"2023-03-25T04:39:23.364593+0000","last_fullsized":"2023-03-25T04:39:23.364593+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:45.608475+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:39:45.608475+0000","last_peered":"2023-03-25T04:39:45.608475+0000","last_clean":"2023-03-25T04:39:45.608475+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:39:45.608475+0000","last_undegraded":"2023-03-25T04:39:45.608475+0000","last_fullsized":"2023-03-25T04:39:45.608475+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":40,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:53.367398+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:39:53.367398+0000","last_peered":"2023-03-25T04:39:53.367398+0000","last_clean":"2023-03-25T04:39:53.367398+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:39:53.367398+0000","last_undegraded":"2023-03-25T04:39:53.367398+0000","last_fullsized":"2023-03-25T04:39:53.367398+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:53.367424+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:39:53.367424+0000","last_peered":"2023-03-25T04:39:53.367424+0000","last_clean":"2023-03-25T04:39:53.367424+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:39:53.367424+0000","last_undegraded":"2023-03-25T04:39:53.367424+0000","last_fullsized":"2023-03-25T04:39:53.367424+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117185,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:43:00 2023","interfaces":[{"interface":"back","average":{"1min":0.27400000000000002,"5min":0.26700000000000002,"15min":0.26600000000000001},"min":{"1min":0.19600000000000001,"5min":0.17499999999999999,"15min":0.16},"max":{"1min":0.42999999999999999,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.27800000000000002},{"interface":"front","average":{"1min":0.28199999999999997,"5min":0.27500000000000002,"15min":0.27300000000000002},"min":{"1min":0.185,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.47199999999999998,"5min":0.47199999999999998,"15min":0.47199999999999998},"last":0.247}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051776,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:42:54 2023","interfaces":[{"interface":"back","average":{"1min":0.26300000000000001,"5min":0.27000000000000002,"15min":0.26900000000000002},"min":{"1min":0.17399999999999999,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.438,"5min":0.44,"15min":0.44},"last":0.20100000000000001},{"interface":"front","average":{"1min":0.27400000000000002,"5min":0.28199999999999997,"15min":0.28299999999999997},"min":{"1min":0.17599999999999999,"5min":0.17599999999999999,"15min":0.16600000000000001},"max":{"1min":0.41599999999999998,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.23499999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:43:25.042+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:43:25.411+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:43:26.013+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:43:47.937+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:43:48.534+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:43:48.889+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:44:09.318 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:44:09.331 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":586,"stamp":"2023-03-25T04:44:08.288933+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002697"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":40,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:40:23.370056+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:40:23.370056+0000","last_peered":"2023-03-25T04:40:23.370056+0000","last_clean":"2023-03-25T04:40:23.370056+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:40:23.370056+0000","last_undegraded":"2023-03-25T04:40:23.370056+0000","last_fullsized":"2023-03-25T04:40:23.370056+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:40:23.370039+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:40:23.370039+0000","last_peered":"2023-03-25T04:40:23.370039+0000","last_clean":"2023-03-25T04:40:23.370039+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:40:23.370039+0000","last_undegraded":"2023-03-25T04:40:23.370039+0000","last_fullsized":"2023-03-25T04:40:23.370039+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:23.375106+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:41:23.375106+0000","last_peered":"2023-03-25T04:41:23.375106+0000","last_clean":"2023-03-25T04:41:23.375106+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:41:23.375106+0000","last_undegraded":"2023-03-25T04:41:23.375106+0000","last_fullsized":"2023-03-25T04:41:23.375106+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:50.618930+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:41:50.618930+0000","last_peered":"2023-03-25T04:41:50.618930+0000","last_clean":"2023-03-25T04:41:50.618930+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:41:50.618930+0000","last_undegraded":"2023-03-25T04:41:50.618930+0000","last_fullsized":"2023-03-25T04:41:50.618930+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":41,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:58.377821+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:41:58.377821+0000","last_peered":"2023-03-25T04:41:58.377821+0000","last_clean":"2023-03-25T04:41:58.377821+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:41:58.377821+0000","last_undegraded":"2023-03-25T04:41:58.377821+0000","last_fullsized":"2023-03-25T04:41:58.377821+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:58.377836+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:41:58.377836+0000","last_peered":"2023-03-25T04:41:58.377836+0000","last_clean":"2023-03-25T04:41:58.377836+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:41:58.377836+0000","last_undegraded":"2023-03-25T04:41:58.377836+0000","last_fullsized":"2023-03-25T04:41:58.377836+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117197,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:44:01 2023","interfaces":[{"interface":"back","average":{"1min":0.27200000000000002,"5min":0.27000000000000002,"15min":0.26400000000000001},"min":{"1min":0.20399999999999999,"5min":0.17499999999999999,"15min":0.16},"max":{"1min":0.38400000000000001,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.245},{"interface":"front","average":{"1min":0.27600000000000002,"5min":0.27800000000000002,"15min":0.27100000000000002},"min":{"1min":0.23100000000000001,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.40400000000000003,"5min":0.47199999999999998,"15min":0.47199999999999998},"last":0.27500000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051788,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:43:55 2023","interfaces":[{"interface":"back","average":{"1min":0.26100000000000001,"5min":0.26700000000000002,"15min":0.27000000000000002},"min":{"1min":0.17799999999999999,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.374,"5min":0.44,"15min":0.44},"last":0.30299999999999999},{"interface":"front","average":{"1min":0.28299999999999997,"5min":0.28100000000000003,"15min":0.28399999999999997},"min":{"1min":0.20899999999999999,"5min":0.17599999999999999,"15min":0.16600000000000001},"max":{"1min":0.40100000000000002,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.27800000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:44:28.923 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":1126,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":19,"modified":"2023-03-25T04:43:20.278959+0000","services":{}},"progress_events":{}}
, 2023-03-25T04:44:28.936 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now
, 2023-03-25T04:44:28.937 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace: 2023-03-25T04:44:48.786 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:44:48.800 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":606,"stamp":"2023-03-25T04:44:48.296805+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002182"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":41,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:42:34.382196+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:42:34.382196+0000","last_peered":"2023-03-25T04:42:34.382196+0000","last_clean":"2023-03-25T04:42:34.382196+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:42:34.382196+0000","last_undegraded":"2023-03-25T04:42:34.382196+0000","last_fullsized":"2023-03-25T04:42:34.382196+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:42:29.381744+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:42:29.381744+0000","last_peered":"2023-03-25T04:42:29.381744+0000","last_clean":"2023-03-25T04:42:29.381744+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:42:29.381744+0000","last_undegraded":"2023-03-25T04:42:29.381744+0000","last_fullsized":"2023-03-25T04:42:29.381744+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:23.375106+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:41:23.375106+0000","last_peered":"2023-03-25T04:41:23.375106+0000","last_clean":"2023-03-25T04:41:23.375106+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:41:23.375106+0000","last_undegraded":"2023-03-25T04:41:23.375106+0000","last_fullsized":"2023-03-25T04:41:23.375106+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:50.618930+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:41:50.618930+0000","last_peered":"2023-03-25T04:41:50.618930+0000","last_clean":"2023-03-25T04:41:50.618930+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:41:50.618930+0000","last_undegraded":"2023-03-25T04:41:50.618930+0000","last_fullsized":"2023-03-25T04:41:50.618930+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":41,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:58.377821+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:41:58.377821+0000","last_peered":"2023-03-25T04:41:58.377821+0000","last_clean":"2023-03-25T04:41:58.377821+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:41:58.377821+0000","last_undegraded":"2023-03-25T04:41:58.377821+0000","last_fullsized":"2023-03-25T04:41:58.377821+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:58.377836+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:41:58.377836+0000","last_peered":"2023-03-25T04:41:58.377836+0000","last_clean":"2023-03-25T04:41:58.377836+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:41:58.377836+0000","last_undegraded":"2023-03-25T04:41:58.377836+0000","last_fullsized":"2023-03-25T04:41:58.377836+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117206,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:44:01 2023","interfaces":[{"interface":"back","average":{"1min":0.27200000000000002,"5min":0.27000000000000002,"15min":0.26400000000000001},"min":{"1min":0.20399999999999999,"5min":0.17499999999999999,"15min":0.16},"max":{"1min":0.38400000000000001,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.216},{"interface":"front","average":{"1min":0.27600000000000002,"5min":0.27800000000000002,"15min":0.27100000000000002},"min":{"1min":0.23100000000000001,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.40400000000000003,"5min":0.47199999999999998,"15min":0.47199999999999998},"last":0.23999999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051796,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:43:55 2023","interfaces":[{"interface":"back","average":{"1min":0.26100000000000001,"5min":0.26700000000000002,"15min":0.27000000000000002},"min":{"1min":0.17799999999999999,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.374,"5min":0.44,"15min":0.44},"last":0.25600000000000001},{"interface":"front","average":{"1min":0.28299999999999997,"5min":0.28100000000000003,"15min":0.28399999999999997},"min":{"1min":0.20899999999999999,"5min":0.17599999999999999,"15min":0.16600000000000001},"max":{"1min":0.40100000000000002,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.28000000000000003}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:45:05.762 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:45:06.073 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all

Ignoring malparsed backtrace: 2023-03-25T04:45:25.658 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":624,"stamp":"2023-03-25T04:45:24.304294+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002366"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":41,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:42:34.382196+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:42:34.382196+0000","last_peered":"2023-03-25T04:42:34.382196+0000","last_clean":"2023-03-25T04:42:34.382196+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:42:34.382196+0000","last_undegraded":"2023-03-25T04:42:34.382196+0000","last_fullsized":"2023-03-25T04:42:34.382196+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:42:29.381744+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:42:29.381744+0000","last_peered":"2023-03-25T04:42:29.381744+0000","last_clean":"2023-03-25T04:42:29.381744+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:42:29.381744+0000","last_undegraded":"2023-03-25T04:42:29.381744+0000","last_fullsized":"2023-03-25T04:42:29.381744+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:23.375106+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:41:23.375106+0000","last_peered":"2023-03-25T04:41:23.375106+0000","last_clean":"2023-03-25T04:41:23.375106+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:41:23.375106+0000","last_undegraded":"2023-03-25T04:41:23.375106+0000","last_fullsized":"2023-03-25T04:41:23.375106+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:50.618930+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:41:50.618930+0000","last_peered":"2023-03-25T04:41:50.618930+0000","last_clean":"2023-03-25T04:41:50.618930+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:41:50.618930+0000","last_undegraded":"2023-03-25T04:41:50.618930+0000","last_fullsized":"2023-03-25T04:41:50.618930+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":41,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:58.377821+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:41:58.377821+0000","last_peered":"2023-03-25T04:41:58.377821+0000","last_clean":"2023-03-25T04:41:58.377821+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:41:58.377821+0000","last_undegraded":"2023-03-25T04:41:58.377821+0000","last_fullsized":"2023-03-25T04:41:58.377821+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:58.377836+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:41:58.377836+0000","last_peered":"2023-03-25T04:41:58.377836+0000","last_clean":"2023-03-25T04:41:58.377836+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:41:58.377836+0000","last_undegraded":"2023-03-25T04:41:58.377836+0000","last_fullsized":"2023-03-25T04:41:58.377836+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117213,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:45:01 2023","interfaces":[{"interface":"back","average":{"1min":0.251,"5min":0.26800000000000002,"15min":0.26400000000000001},"min":{"1min":0.183,"5min":0.17499999999999999,"15min":0.16},"max":{"1min":0.41499999999999998,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.26900000000000002},{"interface":"front","average":{"1min":0.28000000000000003,"5min":0.27900000000000003,"15min":0.27100000000000002},"min":{"1min":0.20999999999999999,"5min":0.185,"15min":0.17499999999999999},"max":{"1min":0.44400000000000001,"5min":0.47199999999999998,"15min":0.47199999999999998},"last":0.27900000000000003}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051803,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:44:55 2023","interfaces":[{"interface":"back","average":{"1min":0.252,"5min":0.26500000000000001,"15min":0.26900000000000002},"min":{"1min":0.183,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.39700000000000002,"5min":0.44,"15min":0.44},"last":0.20699999999999999},{"interface":"front","average":{"1min":0.27000000000000002,"5min":0.28100000000000003,"15min":0.28299999999999997},"min":{"1min":0.20300000000000001,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":0.42099999999999999,"5min":0.47599999999999998,"15min":0.48599999999999999},"last":0.23599999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:45:25.659 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:45:25.969 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:45:42.735+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:45:42.744+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:45:42.897 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace: 2023-03-25T04:46:00.183 INFO:teuthology.orchestra.run.smithi070.stdout:
, 2023-03-25T04:46:00.183 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":1217,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":20,"modified":"2023-03-25T04:45:42.307700+0000","services":{"mgr":{"daemons":{"summary":"","x":{"start_epoch":0,"start_stamp":"0.000000","gid":0,"addr":"(unrecognized address family 0)/0","metadata":{},"task_status":{}}}}}},"progress_events":{}}
, 2023-03-25T04:46:00.197 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now

Ignoring malparsed backtrace: 2023-03-25T04:46:19.910+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:46:20.039 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all

Ignoring malparsed backtrace: 2023-03-25T04:46:39.611 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}
, 2023-03-25T04:46:39.611 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json
, 2023-03-25T04:46:39.613+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:46:58.485+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:46:58.764+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:46:59.519+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:47:14.132 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:47:14.146 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":678,"stamp":"2023-03-25T04:47:12.325044+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002571"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":42,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:44.393426+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:44:44.393426+0000","last_peered":"2023-03-25T04:44:44.393426+0000","last_clean":"2023-03-25T04:44:44.393426+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:44:44.393426+0000","last_undegraded":"2023-03-25T04:44:44.393426+0000","last_fullsized":"2023-03-25T04:44:44.393426+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:39.393044+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:44:39.393044+0000","last_peered":"2023-03-25T04:44:39.393044+0000","last_clean":"2023-03-25T04:44:39.393044+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:44:39.393044+0000","last_undegraded":"2023-03-25T04:44:39.393044+0000","last_fullsized":"2023-03-25T04:44:39.393044+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:43:24.386177+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:43:24.386177+0000","last_peered":"2023-03-25T04:43:24.386177+0000","last_clean":"2023-03-25T04:43:24.386177+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:43:24.386177+0000","last_undegraded":"2023-03-25T04:43:24.386177+0000","last_fullsized":"2023-03-25T04:43:24.386177+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:43:55.629798+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:43:55.629798+0000","last_peered":"2023-03-25T04:43:55.629798+0000","last_clean":"2023-03-25T04:43:55.629798+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:43:55.629798+0000","last_undegraded":"2023-03-25T04:43:55.629798+0000","last_fullsized":"2023-03-25T04:43:55.629798+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":42,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:04.389904+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:44:04.389904+0000","last_peered":"2023-03-25T04:44:04.389904+0000","last_clean":"2023-03-25T04:44:04.389904+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:44:04.389904+0000","last_undegraded":"2023-03-25T04:44:04.389904+0000","last_fullsized":"2023-03-25T04:44:04.389904+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:04.389923+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:44:04.389923+0000","last_peered":"2023-03-25T04:44:04.389923+0000","last_clean":"2023-03-25T04:44:04.389923+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:44:04.389923+0000","last_undegraded":"2023-03-25T04:44:04.389923+0000","last_fullsized":"2023-03-25T04:44:04.389923+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117235,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:47:05 2023","interfaces":[{"interface":"back","average":{"1min":0.27800000000000002,"5min":0.27000000000000002,"15min":0.26300000000000001},"min":{"1min":0.20100000000000001,"5min":0.16900000000000001,"15min":0.16},"max":{"1min":0.39200000000000002,"5min":0.42999999999999999,"15min":0.45600000000000002},"last":0.379},{"interface":"front","average":{"1min":0.28399999999999997,"5min":0.28100000000000003,"15min":0.27200000000000002},"min":{"1min":0.17799999999999999,"5min":0.17799999999999999,"15min":0.17699999999999999},"max":{"1min":0.42599999999999999,"5min":0.47199999999999998,"15min":0.47199999999999998},"last":0.39800000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051825,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:47:01 2023","interfaces":[{"interface":"back","average":{"1min":0.27300000000000002,"5min":0.26100000000000001,"15min":0.27000000000000002},"min":{"1min":0.186,"5min":0.17399999999999999,"15min":0.17000000000000001},"max":{"1min":0.40400000000000003,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.23899999999999999},{"interface":"front","average":{"1min":0.28699999999999998,"5min":0.27500000000000002,"15min":0.28299999999999997},"min":{"1min":0.186,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.48599999999999999},"last":0.19600000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:47:31.198 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json
, 2023-03-25T04:47:31.411+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:47:31.789 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:47:49.509 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":696,"stamp":"2023-03-25T04:47:48.332093+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002697"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":42,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:44.393426+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:44:44.393426+0000","last_peered":"2023-03-25T04:44:44.393426+0000","last_clean":"2023-03-25T04:44:44.393426+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:44:44.393426+0000","last_undegraded":"2023-03-25T04:44:44.393426+0000","last_fullsized":"2023-03-25T04:44:44.393426+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:39.393044+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:44:39.393044+0000","last_peered":"2023-03-25T04:44:39.393044+0000","last_clean":"2023-03-25T04:44:39.393044+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:44:39.393044+0000","last_undegraded":"2023-03-25T04:44:39.393044+0000","last_fullsized":"2023-03-25T04:44:39.393044+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:45:24.396717+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:45:24.396717+0000","last_peered":"2023-03-25T04:45:24.396717+0000","last_clean":"2023-03-25T04:45:24.396717+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:45:24.396717+0000","last_undegraded":"2023-03-25T04:45:24.396717+0000","last_fullsized":"2023-03-25T04:45:24.396717+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:43:55.629798+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:43:55.629798+0000","last_peered":"2023-03-25T04:43:55.629798+0000","last_clean":"2023-03-25T04:43:55.629798+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:43:55.629798+0000","last_undegraded":"2023-03-25T04:43:55.629798+0000","last_fullsized":"2023-03-25T04:43:55.629798+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":42,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:04.389904+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:44:04.389904+0000","last_peered":"2023-03-25T04:44:04.389904+0000","last_clean":"2023-03-25T04:44:04.389904+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:44:04.389904+0000","last_undegraded":"2023-03-25T04:44:04.389904+0000","last_fullsized":"2023-03-25T04:44:04.389904+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:04.389923+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:44:04.389923+0000","last_peered":"2023-03-25T04:44:04.389923+0000","last_clean":"2023-03-25T04:44:04.389923+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:44:04.389923+0000","last_undegraded":"2023-03-25T04:44:04.389923+0000","last_fullsized":"2023-03-25T04:44:04.389923+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117242,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:47:05 2023","interfaces":[{"interface":"back","average":{"1min":0.27800000000000002,"5min":0.27000000000000002,"15min":0.26300000000000001},"min":{"1min":0.20100000000000001,"5min":0.16900000000000001,"15min":0.16},"max":{"1min":0.39200000000000002,"5min":0.42999999999999999,"15min":0.45600000000000002},"last":0.40500000000000003},{"interface":"front","average":{"1min":0.28399999999999997,"5min":0.28100000000000003,"15min":0.27200000000000002},"min":{"1min":0.17799999999999999,"5min":0.17799999999999999,"15min":0.17699999999999999},"max":{"1min":0.42599999999999999,"5min":0.47199999999999998,"15min":0.47199999999999998},"last":0.42399999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051834,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:47:01 2023","interfaces":[{"interface":"back","average":{"1min":0.27300000000000002,"5min":0.26100000000000001,"15min":0.27000000000000002},"min":{"1min":0.186,"5min":0.17399999999999999,"15min":0.17000000000000001},"max":{"1min":0.40400000000000003,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.25600000000000001},{"interface":"front","average":{"1min":0.28699999999999998,"5min":0.27500000000000002,"15min":0.28299999999999997},"min":{"1min":0.186,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.48599999999999999},"last":0.22800000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:47:49.510 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace:     ctx.managers[config['cluster']].wait_for_clean()
,   File "/home/teuthworker/src/github.com_ceph_ceph-c_511015c23eef0684a64171b710e432448df2b7c7/qa/tasks/ceph_manager.py", line 2747, in wait_for_clean
,     assert time.time() - start < timeout, \

Ignoring malparsed backtrace:  ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
,  1: /lib64/libpthread.so.0(+0x12cf0) [0x7f1453c2fcf0]
,  2: __pthread_mutex_lock()

Ignoring malparsed backtrace: 2023-03-25T04:28:05.172 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":142,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":5,"modified":"2023-03-25T04:27:12.092098+0000","services":{}},"progress_events":{}}
, 2023-03-25T04:28:05.185 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now
, 2023-03-25T04:28:05.186 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace: 2023-03-25T04:28:28.360 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":116,"stamp":"2023-03-25T04:28:28.108007+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002578"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.314661+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:27:45.314661+0000","last_peered":"2023-03-25T04:27:45.314661+0000","last_clean":"2023-03-25T04:27:45.314661+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:27:45.314661+0000","last_undegraded":"2023-03-25T04:27:45.314661+0000","last_fullsized":"2023-03-25T04:27:45.314661+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":25,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.310745+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:27:44.310745+0000","last_peered":"2023-03-25T04:27:44.310745+0000","last_clean":"2023-03-25T04:27:44.310745+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:27:44.310745+0000","last_undegraded":"2023-03-25T04:27:44.310745+0000","last_fullsized":"2023-03-25T04:27:44.310745+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":25,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.310650+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:27:44.310650+0000","last_peered":"2023-03-25T04:27:44.310650+0000","last_clean":"2023-03-25T04:27:44.310650+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:27:44.310650+0000","last_undegraded":"2023-03-25T04:27:44.310650+0000","last_fullsized":"2023-03-25T04:27:44.310650+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":26,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.312962+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:27:44.312962+0000","last_peered":"2023-03-25T04:27:44.312962+0000","last_clean":"2023-03-25T04:27:44.312962+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:27:44.312962+0000","last_undegraded":"2023-03-25T04:27:44.312962+0000","last_fullsized":"2023-03-25T04:27:44.312962+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315064+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:27:45.315064+0000","last_peered":"2023-03-25T04:27:45.315064+0000","last_clean":"2023-03-25T04:27:45.315064+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:27:45.315064+0000","last_undegraded":"2023-03-25T04:27:45.315064+0000","last_fullsized":"2023-03-25T04:27:45.315064+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":25,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.310707+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:27:44.310707+0000","last_peered":"2023-03-25T04:27:44.310707+0000","last_clean":"2023-03-25T04:27:44.310707+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:27:44.310707+0000","last_undegraded":"2023-03-25T04:27:44.310707+0000","last_fullsized":"2023-03-25T04:27:44.310707+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117007,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.375}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051597,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.186}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:28:28.362 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean
, 2023-03-25T04:28:28.362 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:28:45.655 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":124,"stamp":"2023-03-25T04:28:44.111265+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002492"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.314661+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:27:45.314661+0000","last_peered":"2023-03-25T04:27:45.314661+0000","last_clean":"2023-03-25T04:27:45.314661+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:27:45.314661+0000","last_undegraded":"2023-03-25T04:27:45.314661+0000","last_fullsized":"2023-03-25T04:27:45.314661+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":25,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.310745+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:27:44.310745+0000","last_peered":"2023-03-25T04:27:44.310745+0000","last_clean":"2023-03-25T04:27:44.310745+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:27:44.310745+0000","last_undegraded":"2023-03-25T04:27:44.310745+0000","last_fullsized":"2023-03-25T04:27:44.310745+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":25,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.310650+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:27:44.310650+0000","last_peered":"2023-03-25T04:27:44.310650+0000","last_clean":"2023-03-25T04:27:44.310650+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:27:44.310650+0000","last_undegraded":"2023-03-25T04:27:44.310650+0000","last_fullsized":"2023-03-25T04:27:44.310650+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":26,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.312962+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:27:44.312962+0000","last_peered":"2023-03-25T04:27:44.312962+0000","last_clean":"2023-03-25T04:27:44.312962+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:27:44.312962+0000","last_undegraded":"2023-03-25T04:27:44.312962+0000","last_fullsized":"2023-03-25T04:27:44.312962+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315064+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:27:45.315064+0000","last_peered":"2023-03-25T04:27:45.315064+0000","last_clean":"2023-03-25T04:27:45.315064+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:27:45.315064+0000","last_undegraded":"2023-03-25T04:27:45.315064+0000","last_fullsized":"2023-03-25T04:27:45.315064+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":25,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.310707+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:27:44.310707+0000","last_peered":"2023-03-25T04:27:44.310707+0000","last_clean":"2023-03-25T04:27:44.310707+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:27:44.310707+0000","last_undegraded":"2023-03-25T04:27:44.310707+0000","last_fullsized":"2023-03-25T04:27:44.310707+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117011,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:28:31 2023","interfaces":[{"interface":"back","average":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.36899999999999999},{"interface":"front","average":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"min":{"1min":0.19,"5min":0.19,"15min":0.19},"max":{"1min":0.46300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.39800000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051601,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:28:24 2023","interfaces":[{"interface":"back","average":{"1min":0.249,"5min":0.249,"15min":0.249},"min":{"1min":0.17699999999999999,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"last":0.254},{"interface":"front","average":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"min":{"1min":0.20200000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"last":0.312}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:28:45.656 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace: 2023-03-25T04:29:05.171 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}
, 2023-03-25T04:29:05.171 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json
, 2023-03-25T04:29:05.561 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:29:22.465 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:29:22.773 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all

Ignoring malparsed backtrace: 2023-03-25T04:29:42.405 INFO:teuthology.orchestra.run.smithi070.stdout:
, 2023-03-25T04:29:42.406 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":240,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":9,"modified":"2023-03-25T04:29:32.120358+0000","services":{"mon":{"daemons":{"summary":"","a":{"start_epoch":0,"start_stamp":"0.000000","gid":0,"addr":"(unrecognized address family 0)/0","metadata":{},"task_status":{}}}}}},"progress_events":{}}
, 2023-03-25T04:29:42.419 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:29:59.627 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":161,"stamp":"2023-03-25T04:29:58.125186+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002398"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315468+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:27:45.315468+0000","last_peered":"2023-03-25T04:27:45.315468+0000","last_clean":"2023-03-25T04:27:45.315468+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:27:45.315468+0000","last_undegraded":"2023-03-25T04:27:45.315468+0000","last_fullsized":"2023-03-25T04:27:45.315468+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312845+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:27:45.312845+0000","last_peered":"2023-03-25T04:27:45.312845+0000","last_clean":"2023-03-25T04:27:45.312845+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:27:45.312845+0000","last_undegraded":"2023-03-25T04:27:45.312845+0000","last_fullsized":"2023-03-25T04:27:45.312845+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312882+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:27:45.312882+0000","last_peered":"2023-03-25T04:27:45.312882+0000","last_clean":"2023-03-25T04:27:45.312882+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:27:45.312882+0000","last_undegraded":"2023-03-25T04:27:45.312882+0000","last_fullsized":"2023-03-25T04:27:45.312882+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.311730+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:27:45.311730+0000","last_peered":"2023-03-25T04:27:45.311730+0000","last_clean":"2023-03-25T04:27:45.311730+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:27:45.311730+0000","last_undegraded":"2023-03-25T04:27:45.311730+0000","last_fullsized":"2023-03-25T04:27:45.311730+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315735+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:27:45.315735+0000","last_peered":"2023-03-25T04:27:45.315735+0000","last_clean":"2023-03-25T04:27:45.315735+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:27:45.315735+0000","last_undegraded":"2023-03-25T04:27:45.315735+0000","last_fullsized":"2023-03-25T04:27:45.315735+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312833+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:27:45.312833+0000","last_peered":"2023-03-25T04:27:45.312833+0000","last_clean":"2023-03-25T04:27:45.312833+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:27:45.312833+0000","last_undegraded":"2023-03-25T04:27:45.312833+0000","last_fullsized":"2023-03-25T04:27:45.312833+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117026,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:29:33 2023","interfaces":[{"interface":"back","average":{"1min":0.26300000000000001,"5min":0.29099999999999998,"15min":0.29599999999999999},"min":{"1min":0.191,"5min":0.191,"15min":0.191},"max":{"1min":0.371,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.20499999999999999},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.29599999999999999,"15min":0.29999999999999999},"min":{"1min":0.21099999999999999,"5min":0.19,"15min":0.19},"max":{"1min":0.39800000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.19500000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051616,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:29:29 2023","interfaces":[{"interface":"back","average":{"1min":0.27200000000000002,"5min":0.253,"15min":0.25},"min":{"1min":0.183,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"last":0.26400000000000001},{"interface":"front","average":{"1min":0.28499999999999998,"5min":0.26800000000000002,"15min":0.26500000000000001},"min":{"1min":0.16600000000000001,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"last":0.29899999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:29:59.629 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean

Ignoring malparsed backtrace: 2023-03-25T04:30:16.889 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:30:16.903 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":170,"stamp":"2023-03-25T04:30:16.128427+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002222"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315468+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:27:45.315468+0000","last_peered":"2023-03-25T04:27:45.315468+0000","last_clean":"2023-03-25T04:27:45.315468+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:27:45.315468+0000","last_undegraded":"2023-03-25T04:27:45.315468+0000","last_fullsized":"2023-03-25T04:27:45.315468+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312845+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:27:45.312845+0000","last_peered":"2023-03-25T04:27:45.312845+0000","last_clean":"2023-03-25T04:27:45.312845+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:27:45.312845+0000","last_undegraded":"2023-03-25T04:27:45.312845+0000","last_fullsized":"2023-03-25T04:27:45.312845+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312882+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:27:45.312882+0000","last_peered":"2023-03-25T04:27:45.312882+0000","last_clean":"2023-03-25T04:27:45.312882+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:27:45.312882+0000","last_undegraded":"2023-03-25T04:27:45.312882+0000","last_fullsized":"2023-03-25T04:27:45.312882+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.311730+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:27:45.311730+0000","last_peered":"2023-03-25T04:27:45.311730+0000","last_clean":"2023-03-25T04:27:45.311730+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:27:45.311730+0000","last_undegraded":"2023-03-25T04:27:45.311730+0000","last_fullsized":"2023-03-25T04:27:45.311730+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315735+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:27:45.315735+0000","last_peered":"2023-03-25T04:27:45.315735+0000","last_clean":"2023-03-25T04:27:45.315735+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:27:45.315735+0000","last_undegraded":"2023-03-25T04:27:45.315735+0000","last_fullsized":"2023-03-25T04:27:45.315735+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312833+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:27:45.312833+0000","last_peered":"2023-03-25T04:27:45.312833+0000","last_clean":"2023-03-25T04:27:45.312833+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:27:45.312833+0000","last_undegraded":"2023-03-25T04:27:45.312833+0000","last_fullsized":"2023-03-25T04:27:45.312833+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117030,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:29:33 2023","interfaces":[{"interface":"back","average":{"1min":0.26300000000000001,"5min":0.29099999999999998,"15min":0.29599999999999999},"min":{"1min":0.191,"5min":0.191,"15min":0.191},"max":{"1min":0.371,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.245},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.29599999999999999,"15min":0.29999999999999999},"min":{"1min":0.21099999999999999,"5min":0.19,"15min":0.19},"max":{"1min":0.39800000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.20899999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051620,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:29:29 2023","interfaces":[{"interface":"back","average":{"1min":0.27200000000000002,"5min":0.253,"15min":0.25},"min":{"1min":0.183,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"last":0.19400000000000001},{"interface":"front","average":{"1min":0.28499999999999998,"5min":0.26800000000000002,"15min":0.26500000000000001},"min":{"1min":0.16600000000000001,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"last":0.23599999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:30:31.984+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:30:32.201+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:30:32.981+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)

Ignoring malparsed backtrace: 2023-03-25T04:30:48.449 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now
, 2023-03-25T04:30:48.450 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:30:48.760 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:31:08.342 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":196,"stamp":"2023-03-25T04:31:08.139018+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002431"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315468+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:27:45.315468+0000","last_peered":"2023-03-25T04:27:45.315468+0000","last_clean":"2023-03-25T04:27:45.315468+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:27:45.315468+0000","last_undegraded":"2023-03-25T04:27:45.315468+0000","last_fullsized":"2023-03-25T04:27:45.315468+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312845+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:27:45.312845+0000","last_peered":"2023-03-25T04:27:45.312845+0000","last_clean":"2023-03-25T04:27:45.312845+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:27:45.312845+0000","last_undegraded":"2023-03-25T04:27:45.312845+0000","last_fullsized":"2023-03-25T04:27:45.312845+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312882+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:27:45.312882+0000","last_peered":"2023-03-25T04:27:45.312882+0000","last_clean":"2023-03-25T04:27:45.312882+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:27:45.312882+0000","last_undegraded":"2023-03-25T04:27:45.312882+0000","last_fullsized":"2023-03-25T04:27:45.312882+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.311730+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:27:45.311730+0000","last_peered":"2023-03-25T04:27:45.311730+0000","last_clean":"2023-03-25T04:27:45.311730+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:27:45.311730+0000","last_undegraded":"2023-03-25T04:27:45.311730+0000","last_fullsized":"2023-03-25T04:27:45.311730+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315735+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:27:45.315735+0000","last_peered":"2023-03-25T04:27:45.315735+0000","last_clean":"2023-03-25T04:27:45.315735+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:27:45.315735+0000","last_undegraded":"2023-03-25T04:27:45.315735+0000","last_fullsized":"2023-03-25T04:27:45.315735+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312833+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:27:45.312833+0000","last_peered":"2023-03-25T04:27:45.312833+0000","last_clean":"2023-03-25T04:27:45.312833+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:27:45.312833+0000","last_undegraded":"2023-03-25T04:27:45.312833+0000","last_fullsized":"2023-03-25T04:27:45.312833+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117040,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:30:37 2023","interfaces":[{"interface":"back","average":{"1min":0.28799999999999998,"5min":0.28899999999999998,"15min":0.29499999999999998},"min":{"1min":0.20200000000000001,"5min":0.191,"15min":0.191},"max":{"1min":0.433,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.29799999999999999},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.28999999999999998,"15min":0.29799999999999999},"min":{"1min":0.17499999999999999,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":0.40300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.25700000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051630,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:30:32 2023","interfaces":[{"interface":"back","average":{"1min":0.25600000000000001,"5min":0.255,"15min":0.251},"min":{"1min":0.19400000000000001,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.35699999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"last":0.22800000000000001},{"interface":"front","average":{"1min":0.28699999999999998,"5min":0.27200000000000002,"15min":0.26600000000000001},"min":{"1min":0.20599999999999999,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.39400000000000002,"5min":0.42899999999999999,"15min":0.42899999999999999},"last":0.24099999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:31:08.343 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace: 2023-03-25T04:31:24.283+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:31:25.192+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:31:25.295+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)

Ignoring malparsed backtrace: 2023-03-25T04:31:40.257 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:31:40.271 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":212,"stamp":"2023-03-25T04:31:40.144732+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002208"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315468+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:27:45.315468+0000","last_peered":"2023-03-25T04:27:45.315468+0000","last_clean":"2023-03-25T04:27:45.315468+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:27:45.315468+0000","last_undegraded":"2023-03-25T04:27:45.315468+0000","last_fullsized":"2023-03-25T04:27:45.315468+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312845+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:27:45.312845+0000","last_peered":"2023-03-25T04:27:45.312845+0000","last_clean":"2023-03-25T04:27:45.312845+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:27:45.312845+0000","last_undegraded":"2023-03-25T04:27:45.312845+0000","last_fullsized":"2023-03-25T04:27:45.312845+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:23.308607+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:29:23.308607+0000","last_peered":"2023-03-25T04:29:23.308607+0000","last_clean":"2023-03-25T04:29:23.308607+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:29:23.308607+0000","last_undegraded":"2023-03-25T04:29:23.308607+0000","last_fullsized":"2023-03-25T04:29:23.308607+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":28,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:20.554710+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:29:20.554710+0000","last_peered":"2023-03-25T04:29:20.554710+0000","last_clean":"2023-03-25T04:29:20.554710+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:29:20.554710+0000","last_undegraded":"2023-03-25T04:29:20.554710+0000","last_fullsized":"2023-03-25T04:29:20.554710+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309216+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:29:28.309216+0000","last_peered":"2023-03-25T04:29:28.309216+0000","last_clean":"2023-03-25T04:29:28.309216+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:29:28.309216+0000","last_undegraded":"2023-03-25T04:29:28.309216+0000","last_fullsized":"2023-03-25T04:29:28.309216+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309233+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:29:28.309233+0000","last_peered":"2023-03-25T04:29:28.309233+0000","last_clean":"2023-03-25T04:29:28.309233+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:29:28.309233+0000","last_undegraded":"2023-03-25T04:29:28.309233+0000","last_fullsized":"2023-03-25T04:29:28.309233+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117046,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:30:37 2023","interfaces":[{"interface":"back","average":{"1min":0.28799999999999998,"5min":0.28899999999999998,"15min":0.29499999999999998},"min":{"1min":0.20200000000000001,"5min":0.191,"15min":0.191},"max":{"1min":0.433,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.216},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.28999999999999998,"15min":0.29799999999999999},"min":{"1min":0.17499999999999999,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":0.40300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.24299999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051637,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:31:35 2023","interfaces":[{"interface":"back","average":{"1min":0.26100000000000001,"5min":0.25700000000000001,"15min":0.251},"min":{"1min":0.19,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.35899999999999999,"5min":0.41599999999999998,"15min":0.41599999999999998},"last":0.221},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.27400000000000002,"15min":0.26700000000000002},"min":{"1min":0.20200000000000001,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.374,"5min":0.42899999999999999,"15min":0.42899999999999999},"last":0.24099999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:32:01.906+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:32:02.388+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:32:02.794 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:32:22.326 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":233,"stamp":"2023-03-25T04:32:22.153289+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002545"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:33.309748+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:29:33.309748+0000","last_peered":"2023-03-25T04:29:33.309748+0000","last_clean":"2023-03-25T04:29:33.309748+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:29:33.309748+0000","last_undegraded":"2023-03-25T04:29:33.309748+0000","last_fullsized":"2023-03-25T04:29:33.309748+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:33.309729+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:29:33.309729+0000","last_peered":"2023-03-25T04:29:33.309729+0000","last_clean":"2023-03-25T04:29:33.309729+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:29:33.309729+0000","last_undegraded":"2023-03-25T04:29:33.309729+0000","last_fullsized":"2023-03-25T04:29:33.309729+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:23.308607+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:29:23.308607+0000","last_peered":"2023-03-25T04:29:23.308607+0000","last_clean":"2023-03-25T04:29:23.308607+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:29:23.308607+0000","last_undegraded":"2023-03-25T04:29:23.308607+0000","last_fullsized":"2023-03-25T04:29:23.308607+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":28,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:20.554710+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:29:20.554710+0000","last_peered":"2023-03-25T04:29:20.554710+0000","last_clean":"2023-03-25T04:29:20.554710+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:29:20.554710+0000","last_undegraded":"2023-03-25T04:29:20.554710+0000","last_fullsized":"2023-03-25T04:29:20.554710+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309216+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:29:28.309216+0000","last_peered":"2023-03-25T04:29:28.309216+0000","last_clean":"2023-03-25T04:29:28.309216+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:29:28.309216+0000","last_undegraded":"2023-03-25T04:29:28.309216+0000","last_fullsized":"2023-03-25T04:29:28.309216+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309233+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:29:28.309233+0000","last_peered":"2023-03-25T04:29:28.309233+0000","last_clean":"2023-03-25T04:29:28.309233+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:29:28.309233+0000","last_undegraded":"2023-03-25T04:29:28.309233+0000","last_fullsized":"2023-03-25T04:29:28.309233+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117055,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:31:42 2023","interfaces":[{"interface":"back","average":{"1min":0.27500000000000002,"5min":0.28399999999999997,"15min":0.29399999999999998},"min":{"1min":0.19800000000000001,"5min":0.191,"15min":0.191},"max":{"1min":0.36899999999999999,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.32300000000000001},{"interface":"front","average":{"1min":0.27900000000000003,"5min":0.28499999999999998,"15min":0.29599999999999999},"min":{"1min":0.216,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":0.40799999999999997,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.27100000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051646,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:31:35 2023","interfaces":[{"interface":"back","average":{"1min":0.26100000000000001,"5min":0.25700000000000001,"15min":0.251},"min":{"1min":0.19,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.35899999999999999,"5min":0.41599999999999998,"15min":0.41599999999999998},"last":0.25600000000000001},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.27400000000000002,"15min":0.26700000000000002},"min":{"1min":0.20200000000000001,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.374,"5min":0.42899999999999999,"15min":0.42899999999999999},"last":0.29199999999999998}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:32:22.327 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean
, 2023-03-25T04:32:22.327 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}

Ignoring malparsed backtrace: 2023-03-25T04:32:39.836 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:32:39.849 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":241,"stamp":"2023-03-25T04:32:38.156312+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002346"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:33.309748+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:29:33.309748+0000","last_peered":"2023-03-25T04:29:33.309748+0000","last_clean":"2023-03-25T04:29:33.309748+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:29:33.309748+0000","last_undegraded":"2023-03-25T04:29:33.309748+0000","last_fullsized":"2023-03-25T04:29:33.309748+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:33.309729+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:29:33.309729+0000","last_peered":"2023-03-25T04:29:33.309729+0000","last_clean":"2023-03-25T04:29:33.309729+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:29:33.309729+0000","last_undegraded":"2023-03-25T04:29:33.309729+0000","last_fullsized":"2023-03-25T04:29:33.309729+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:23.308607+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:29:23.308607+0000","last_peered":"2023-03-25T04:29:23.308607+0000","last_clean":"2023-03-25T04:29:23.308607+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:29:23.308607+0000","last_undegraded":"2023-03-25T04:29:23.308607+0000","last_fullsized":"2023-03-25T04:29:23.308607+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":28,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:20.554710+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:29:20.554710+0000","last_peered":"2023-03-25T04:29:20.554710+0000","last_clean":"2023-03-25T04:29:20.554710+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:29:20.554710+0000","last_undegraded":"2023-03-25T04:29:20.554710+0000","last_fullsized":"2023-03-25T04:29:20.554710+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309216+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:29:28.309216+0000","last_peered":"2023-03-25T04:29:28.309216+0000","last_clean":"2023-03-25T04:29:28.309216+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:29:28.309216+0000","last_undegraded":"2023-03-25T04:29:28.309216+0000","last_fullsized":"2023-03-25T04:29:28.309216+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309233+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:29:28.309233+0000","last_peered":"2023-03-25T04:29:28.309233+0000","last_clean":"2023-03-25T04:29:28.309233+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:29:28.309233+0000","last_undegraded":"2023-03-25T04:29:28.309233+0000","last_fullsized":"2023-03-25T04:29:28.309233+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117058,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:31:42 2023","interfaces":[{"interface":"back","average":{"1min":0.27500000000000002,"5min":0.28399999999999997,"15min":0.29399999999999998},"min":{"1min":0.19800000000000001,"5min":0.191,"15min":0.191},"max":{"1min":0.36899999999999999,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.30599999999999999},{"interface":"front","average":{"1min":0.27900000000000003,"5min":0.28499999999999998,"15min":0.29599999999999999},"min":{"1min":0.216,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":0.40799999999999997,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.32500000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051649,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:31:35 2023","interfaces":[{"interface":"back","average":{"1min":0.26100000000000001,"5min":0.25700000000000001,"15min":0.251},"min":{"1min":0.19,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.35899999999999999,"5min":0.41599999999999998,"15min":0.41599999999999998},"last":0.24199999999999999},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.27400000000000002,"15min":0.26700000000000002},"min":{"1min":0.20200000000000001,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.374,"5min":0.42899999999999999,"15min":0.42899999999999999},"last":0.22600000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:32:54.944+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:32:55.469+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:32:55.927+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)

Ignoring malparsed backtrace: 2023-03-25T04:33:13.958 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean
, 2023-03-25T04:33:13.959 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}
, 2023-03-25T04:33:13.959 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json

Ignoring malparsed backtrace: 2023-03-25T04:33:31.199 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":267,"stamp":"2023-03-25T04:33:30.166660+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002269"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:33.309748+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:29:33.309748+0000","last_peered":"2023-03-25T04:29:33.309748+0000","last_clean":"2023-03-25T04:29:33.309748+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:29:33.309748+0000","last_undegraded":"2023-03-25T04:29:33.309748+0000","last_fullsized":"2023-03-25T04:29:33.309748+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:33.309729+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:29:33.309729+0000","last_peered":"2023-03-25T04:29:33.309729+0000","last_clean":"2023-03-25T04:29:33.309729+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:29:33.309729+0000","last_undegraded":"2023-03-25T04:29:33.309729+0000","last_fullsized":"2023-03-25T04:29:33.309729+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":28,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:31:23.320894+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:31:23.320894+0000","last_peered":"2023-03-25T04:31:23.320894+0000","last_clean":"2023-03-25T04:31:23.320894+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:31:23.320894+0000","last_undegraded":"2023-03-25T04:31:23.320894+0000","last_fullsized":"2023-03-25T04:31:23.320894+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":28,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:20.554710+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:29:20.554710+0000","last_peered":"2023-03-25T04:29:20.554710+0000","last_clean":"2023-03-25T04:29:20.554710+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:29:20.554710+0000","last_undegraded":"2023-03-25T04:29:20.554710+0000","last_fullsized":"2023-03-25T04:29:20.554710+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309216+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:29:28.309216+0000","last_peered":"2023-03-25T04:29:28.309216+0000","last_clean":"2023-03-25T04:29:28.309216+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:29:28.309216+0000","last_undegraded":"2023-03-25T04:29:28.309216+0000","last_fullsized":"2023-03-25T04:29:28.309216+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309233+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:29:28.309233+0000","last_peered":"2023-03-25T04:29:28.309233+0000","last_clean":"2023-03-25T04:29:28.309233+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:29:28.309233+0000","last_undegraded":"2023-03-25T04:29:28.309233+0000","last_fullsized":"2023-03-25T04:29:28.309233+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117068,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:32:46 2023","interfaces":[{"interface":"back","average":{"1min":0.27000000000000002,"5min":0.27900000000000003,"15min":0.29199999999999998},"min":{"1min":0.17899999999999999,"5min":0.17899999999999999,"15min":0.17899999999999999},"max":{"1min":0.35599999999999998,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.28799999999999998},{"interface":"front","average":{"1min":0.26800000000000002,"5min":0.27800000000000002,"15min":0.29399999999999998},"min":{"1min":0.19900000000000001,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":0.377,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.315}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051660,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:32:36 2023","interfaces":[{"interface":"back","average":{"1min":0.248,"5min":0.25700000000000001,"15min":0.251},"min":{"1min":0.17000000000000001,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"last":0.22700000000000001},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.27600000000000002,"15min":0.26800000000000002},"min":{"1min":0.192,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"last":0.20399999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:33:31.201 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:33:31.400+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)

Ignoring malparsed backtrace: 2023-03-25T04:33:47.531+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:33:47.951+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:33:48.505+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)

Ignoring malparsed backtrace: 2023-03-25T04:34:06.104 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json
, 2023-03-25T04:34:06.118+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:34:06.497 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:34:23.399 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:34:23.665+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:34:23.712 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:34:40.195+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:34:40.523+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:34:40.676 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace: 2023-03-25T04:35:03.381+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:35:04.139+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:35:04.391+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)

Ignoring malparsed backtrace: 2023-03-25T04:35:22.738 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean
, 2023-03-25T04:35:22.739 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}
, 2023-03-25T04:35:22.739 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json

Ignoring malparsed backtrace: 2023-03-25T04:35:41.393+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:35:42.352+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:35:42.376+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:35:57.384 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":340,"stamp":"2023-03-25T04:35:56.194992+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.001930"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":36,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:31:43.322894+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:31:43.322894+0000","last_peered":"2023-03-25T04:31:43.322894+0000","last_clean":"2023-03-25T04:31:43.322894+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:31:43.322894+0000","last_undegraded":"2023-03-25T04:31:43.322894+0000","last_fullsized":"2023-03-25T04:31:43.322894+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":28,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:31:43.322876+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:31:43.322876+0000","last_peered":"2023-03-25T04:31:43.322876+0000","last_clean":"2023-03-25T04:31:43.322876+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:31:43.322876+0000","last_undegraded":"2023-03-25T04:31:43.322876+0000","last_fullsized":"2023-03-25T04:31:43.322876+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:23.331998+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:33:23.331998+0000","last_peered":"2023-03-25T04:33:23.331998+0000","last_clean":"2023-03-25T04:33:23.331998+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:33:23.331998+0000","last_undegraded":"2023-03-25T04:33:23.331998+0000","last_fullsized":"2023-03-25T04:33:23.331998+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:30.576723+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:33:30.576723+0000","last_peered":"2023-03-25T04:33:30.576723+0000","last_clean":"2023-03-25T04:33:30.576723+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:33:30.576723+0000","last_undegraded":"2023-03-25T04:33:30.576723+0000","last_fullsized":"2023-03-25T04:33:30.576723+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":37,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:38.333447+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:33:38.333447+0000","last_peered":"2023-03-25T04:33:38.333447+0000","last_clean":"2023-03-25T04:33:38.333447+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:33:38.333447+0000","last_undegraded":"2023-03-25T04:33:38.333447+0000","last_fullsized":"2023-03-25T04:33:38.333447+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:38.333467+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:33:38.333467+0000","last_peered":"2023-03-25T04:33:38.333467+0000","last_clean":"2023-03-25T04:33:38.333467+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:33:38.333467+0000","last_undegraded":"2023-03-25T04:33:38.333467+0000","last_fullsized":"2023-03-25T04:33:38.333467+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117098,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:35:49 2023","interfaces":[{"interface":"back","average":{"1min":0.255,"5min":0.26000000000000001,"15min":0.28199999999999997},"min":{"1min":0.17899999999999999,"5min":0.16,"15min":0.16},"max":{"1min":0.36399999999999999,"5min":0.42599999999999999,"15min":0.45100000000000001},"last":0.26100000000000001},{"interface":"front","average":{"1min":0.26000000000000001,"5min":0.26700000000000002,"15min":0.28599999999999998},"min":{"1min":0.188,"5min":0.17999999999999999,"15min":0.17499999999999999},"max":{"1min":0.39100000000000001,"5min":0.45200000000000001,"15min":0.46300000000000002},"last":0.28399999999999997}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051689,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:35:42 2023","interfaces":[{"interface":"back","average":{"1min":0.29399999999999998,"5min":0.27200000000000002,"15min":0.25800000000000001},"min":{"1min":0.17299999999999999,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.41099999999999998,"5min":0.42699999999999999,"15min":0.42699999999999999},"last":0.20399999999999999},{"interface":"front","average":{"1min":0.30299999999999999,"5min":0.28399999999999997,"15min":0.27300000000000002},"min":{"1min":0.19700000000000001,"5min":0.192,"15min":0.16600000000000001},"max":{"1min":0.43099999999999999,"5min":0.45800000000000002,"15min":0.45800000000000002},"last":0.22}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:35:57.624+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)

Ignoring malparsed backtrace: 2023-03-25T04:36:17.226 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:36:17.239 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":350,"stamp":"2023-03-25T04:36:16.198866+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002148"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":37,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:53.335026+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:33:53.335026+0000","last_peered":"2023-03-25T04:33:53.335026+0000","last_clean":"2023-03-25T04:33:53.335026+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:33:53.335026+0000","last_undegraded":"2023-03-25T04:33:53.335026+0000","last_fullsized":"2023-03-25T04:33:53.335026+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:53.335009+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:33:53.335009+0000","last_peered":"2023-03-25T04:33:53.335009+0000","last_clean":"2023-03-25T04:33:53.335009+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:33:53.335009+0000","last_undegraded":"2023-03-25T04:33:53.335009+0000","last_fullsized":"2023-03-25T04:33:53.335009+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:23.331998+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:33:23.331998+0000","last_peered":"2023-03-25T04:33:23.331998+0000","last_clean":"2023-03-25T04:33:23.331998+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:33:23.331998+0000","last_undegraded":"2023-03-25T04:33:23.331998+0000","last_fullsized":"2023-03-25T04:33:23.331998+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:30.576723+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:33:30.576723+0000","last_peered":"2023-03-25T04:33:30.576723+0000","last_clean":"2023-03-25T04:33:30.576723+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:33:30.576723+0000","last_undegraded":"2023-03-25T04:33:30.576723+0000","last_fullsized":"2023-03-25T04:33:30.576723+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":37,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:38.333447+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:33:38.333447+0000","last_peered":"2023-03-25T04:33:38.333447+0000","last_clean":"2023-03-25T04:33:38.333447+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:33:38.333447+0000","last_undegraded":"2023-03-25T04:33:38.333447+0000","last_fullsized":"2023-03-25T04:33:38.333447+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:38.333467+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:33:38.333467+0000","last_peered":"2023-03-25T04:33:38.333467+0000","last_clean":"2023-03-25T04:33:38.333467+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:33:38.333467+0000","last_undegraded":"2023-03-25T04:33:38.333467+0000","last_fullsized":"2023-03-25T04:33:38.333467+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117102,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:35:49 2023","interfaces":[{"interface":"back","average":{"1min":0.255,"5min":0.26000000000000001,"15min":0.28199999999999997},"min":{"1min":0.17899999999999999,"5min":0.16,"15min":0.16},"max":{"1min":0.36399999999999999,"5min":0.42599999999999999,"15min":0.45100000000000001},"last":0.28399999999999997},{"interface":"front","average":{"1min":0.26000000000000001,"5min":0.26700000000000002,"15min":0.28599999999999998},"min":{"1min":0.188,"5min":0.17999999999999999,"15min":0.17499999999999999},"max":{"1min":0.39100000000000001,"5min":0.45200000000000001,"15min":0.46300000000000002},"last":0.30299999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051693,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:35:42 2023","interfaces":[{"interface":"back","average":{"1min":0.29399999999999998,"5min":0.27200000000000002,"15min":0.25800000000000001},"min":{"1min":0.17299999999999999,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.41099999999999998,"5min":0.42699999999999999,"15min":0.42699999999999999},"last":0.35099999999999998},{"interface":"front","average":{"1min":0.30299999999999999,"5min":0.28399999999999997,"15min":0.27300000000000002},"min":{"1min":0.19700000000000001,"5min":0.192,"15min":0.16600000000000001},"max":{"1min":0.43099999999999999,"5min":0.45800000000000002,"15min":0.45800000000000002},"last":0.32900000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:36:34.556+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:36:34.717+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:36:35.583+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)

Ignoring malparsed backtrace: 2023-03-25T04:36:52.680 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json
, 2023-03-25T04:36:52.701+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:36:53.067 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:37:09.963 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:37:10.273 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all

Ignoring malparsed backtrace: 2023-03-25T04:37:26.856+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:37:27.288 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:37:27.538+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:37:41.903 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":392,"stamp":"2023-03-25T04:37:40.213373+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002012"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":37,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:53.335026+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:33:53.335026+0000","last_peered":"2023-03-25T04:33:53.335026+0000","last_clean":"2023-03-25T04:33:53.335026+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:33:53.335026+0000","last_undegraded":"2023-03-25T04:33:53.335026+0000","last_fullsized":"2023-03-25T04:33:53.335026+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:53.335009+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:33:53.335009+0000","last_peered":"2023-03-25T04:33:53.335009+0000","last_clean":"2023-03-25T04:33:53.335009+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:33:53.335009+0000","last_undegraded":"2023-03-25T04:33:53.335009+0000","last_fullsized":"2023-03-25T04:33:53.335009+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:23.343585+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:35:23.343585+0000","last_peered":"2023-03-25T04:35:23.343585+0000","last_clean":"2023-03-25T04:35:23.343585+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:35:23.343585+0000","last_undegraded":"2023-03-25T04:35:23.343585+0000","last_fullsized":"2023-03-25T04:35:23.343585+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:30.576723+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:33:30.576723+0000","last_peered":"2023-03-25T04:33:30.576723+0000","last_clean":"2023-03-25T04:33:30.576723+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:33:30.576723+0000","last_undegraded":"2023-03-25T04:33:30.576723+0000","last_fullsized":"2023-03-25T04:33:30.576723+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":37,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:38.333447+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:33:38.333447+0000","last_peered":"2023-03-25T04:33:38.333447+0000","last_clean":"2023-03-25T04:33:38.333447+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:33:38.333447+0000","last_undegraded":"2023-03-25T04:33:38.333447+0000","last_fullsized":"2023-03-25T04:33:38.333447+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:38.333467+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:33:38.333467+0000","last_peered":"2023-03-25T04:33:38.333467+0000","last_clean":"2023-03-25T04:33:38.333467+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:33:38.333467+0000","last_undegraded":"2023-03-25T04:33:38.333467+0000","last_fullsized":"2023-03-25T04:33:38.333467+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117119,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:36:50 2023","interfaces":[{"interface":"back","average":{"1min":0.254,"5min":0.25600000000000001,"15min":0.27900000000000003},"min":{"1min":0.20499999999999999,"5min":0.16,"15min":0.16},"max":{"1min":0.371,"5min":0.42599999999999999,"15min":0.45100000000000001},"last":0.23000000000000001},{"interface":"front","average":{"1min":0.26800000000000002,"5min":0.26500000000000001,"15min":0.28399999999999997},"min":{"1min":0.19700000000000001,"5min":0.17999999999999999,"15min":0.17499999999999999},"max":{"1min":0.34699999999999998,"5min":0.45200000000000001,"15min":0.46300000000000002},"last":0.20899999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051710,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:36:44 2023","interfaces":[{"interface":"back","average":{"1min":0.28399999999999997,"5min":0.27700000000000002,"15min":0.26100000000000001},"min":{"1min":0.192,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.40899999999999997,"5min":0.42699999999999999,"15min":0.42699999999999999},"last":0.39700000000000002},{"interface":"front","average":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.27500000000000002},"min":{"1min":0.182,"5min":0.182,"15min":0.16600000000000001},"max":{"1min":0.439,"5min":0.45800000000000002,"15min":0.45800000000000002},"last":0.439}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:37:42.400+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)

Ignoring malparsed backtrace: 2023-03-25T04:38:04.766+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:38:04.922 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:38:05.246 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:38:19.605 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":411,"stamp":"2023-03-25T04:38:18.220966+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002368"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":38,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:36:03.347070+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:36:03.347070+0000","last_peered":"2023-03-25T04:36:03.347070+0000","last_clean":"2023-03-25T04:36:03.347070+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:36:03.347070+0000","last_undegraded":"2023-03-25T04:36:03.347070+0000","last_fullsized":"2023-03-25T04:36:03.347070+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:36:03.347025+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:36:03.347025+0000","last_peered":"2023-03-25T04:36:03.347025+0000","last_clean":"2023-03-25T04:36:03.347025+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:36:03.347025+0000","last_undegraded":"2023-03-25T04:36:03.347025+0000","last_fullsized":"2023-03-25T04:36:03.347025+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:23.343585+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:35:23.343585+0000","last_peered":"2023-03-25T04:35:23.343585+0000","last_clean":"2023-03-25T04:35:23.343585+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:35:23.343585+0000","last_undegraded":"2023-03-25T04:35:23.343585+0000","last_fullsized":"2023-03-25T04:35:23.343585+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:35.587025+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:35:35.587025+0000","last_peered":"2023-03-25T04:35:35.587025+0000","last_clean":"2023-03-25T04:35:35.587025+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:35:35.587025+0000","last_undegraded":"2023-03-25T04:35:35.587025+0000","last_fullsized":"2023-03-25T04:35:35.587025+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":38,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:43.345308+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:35:43.345308+0000","last_peered":"2023-03-25T04:35:43.345308+0000","last_clean":"2023-03-25T04:35:43.345308+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:35:43.345308+0000","last_undegraded":"2023-03-25T04:35:43.345308+0000","last_fullsized":"2023-03-25T04:35:43.345308+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:43.345324+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:35:43.345324+0000","last_peered":"2023-03-25T04:35:43.345324+0000","last_clean":"2023-03-25T04:35:43.345324+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:35:43.345324+0000","last_undegraded":"2023-03-25T04:35:43.345324+0000","last_fullsized":"2023-03-25T04:35:43.345324+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117127,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:37:53 2023","interfaces":[{"interface":"back","average":{"1min":0.25800000000000001,"5min":0.253,"15min":0.27700000000000002},"min":{"1min":0.184,"5min":0.16,"15min":0.16},"max":{"1min":0.44600000000000001,"5min":0.44600000000000001,"15min":0.45100000000000001},"last":0.35199999999999998},{"interface":"front","average":{"1min":0.26900000000000002,"5min":0.26500000000000001,"15min":0.28199999999999997},"min":{"1min":0.192,"5min":0.17999999999999999,"15min":0.17499999999999999},"max":{"1min":0.46400000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"last":0.33200000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051717,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:37:44 2023","interfaces":[{"interface":"back","average":{"1min":0.27300000000000002,"5min":0.28199999999999997,"15min":0.26200000000000001},"min":{"1min":0.17899999999999999,"5min":0.17299999999999999,"15min":0.17000000000000001},"max":{"1min":0.41799999999999998,"5min":0.42099999999999999,"15min":0.42699999999999999},"last":0.28699999999999998},{"interface":"front","average":{"1min":0.29399999999999998,"5min":0.29099999999999998,"15min":0.27700000000000002},"min":{"1min":0.19900000000000001,"5min":0.182,"15min":0.16600000000000001},"max":{"1min":0.441,"5min":0.441,"15min":0.45800000000000002},"last":0.26300000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:38:19.718+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:38:20.315+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)

Ignoring malparsed backtrace: 2023-03-25T04:38:44.283 INFO:teuthology.orchestra.run.smithi070.stdout:
, 2023-03-25T04:38:44.283 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":781,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":16,"modified":"2023-03-25T04:38:40.225321+0000","services":{}},"progress_events":{}}
, 2023-03-25T04:38:44.296 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now

Ignoring malparsed backtrace: 2023-03-25T04:39:02.477+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:39:02.578+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:39:03.524+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:39:20.359 INFO:teuthology.orchestra.run.smithi070.stdout:
, 2023-03-25T04:39:20.359 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":818,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":16,"modified":"2023-03-25T04:38:40.225321+0000","services":{}},"progress_events":{}}
, 2023-03-25T04:39:20.372 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now

Ignoring malparsed backtrace: 2023-03-25T04:39:37.527 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:39:37.541 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":450,"stamp":"2023-03-25T04:39:36.235332+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002086"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":38,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:36:03.347070+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:36:03.347070+0000","last_peered":"2023-03-25T04:36:03.347070+0000","last_clean":"2023-03-25T04:36:03.347070+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:36:03.347070+0000","last_undegraded":"2023-03-25T04:36:03.347070+0000","last_fullsized":"2023-03-25T04:36:03.347070+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:36:03.347025+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:36:03.347025+0000","last_peered":"2023-03-25T04:36:03.347025+0000","last_clean":"2023-03-25T04:36:03.347025+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:36:03.347025+0000","last_undegraded":"2023-03-25T04:36:03.347025+0000","last_fullsized":"2023-03-25T04:36:03.347025+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:23.353778+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:37:23.353778+0000","last_peered":"2023-03-25T04:37:23.353778+0000","last_clean":"2023-03-25T04:37:23.353778+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:37:23.353778+0000","last_undegraded":"2023-03-25T04:37:23.353778+0000","last_fullsized":"2023-03-25T04:37:23.353778+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:35.587025+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:35:35.587025+0000","last_peered":"2023-03-25T04:35:35.587025+0000","last_clean":"2023-03-25T04:35:35.587025+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:35:35.587025+0000","last_undegraded":"2023-03-25T04:35:35.587025+0000","last_fullsized":"2023-03-25T04:35:35.587025+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":38,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:43.345308+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:35:43.345308+0000","last_peered":"2023-03-25T04:35:43.345308+0000","last_clean":"2023-03-25T04:35:43.345308+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:35:43.345308+0000","last_undegraded":"2023-03-25T04:35:43.345308+0000","last_fullsized":"2023-03-25T04:35:43.345308+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:43.345324+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:35:43.345324+0000","last_peered":"2023-03-25T04:35:43.345324+0000","last_clean":"2023-03-25T04:35:43.345324+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:35:43.345324+0000","last_undegraded":"2023-03-25T04:35:43.345324+0000","last_fullsized":"2023-03-25T04:35:43.345324+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117143,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:38:54 2023","interfaces":[{"interface":"back","average":{"1min":0.25800000000000001,"5min":0.25700000000000001,"15min":0.27400000000000002},"min":{"1min":0.192,"5min":0.17399999999999999,"15min":0.16},"max":{"1min":0.39400000000000002,"5min":0.44600000000000001,"15min":0.45100000000000001},"last":0.26500000000000001},{"interface":"front","average":{"1min":0.25900000000000001,"5min":0.26700000000000002,"15min":0.27900000000000003},"min":{"1min":0.19500000000000001,"5min":0.188,"15min":0.17499999999999999},"max":{"1min":0.41499999999999998,"5min":0.46400000000000002,"15min":0.46400000000000002},"last":0.29099999999999998}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051733,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:38:45 2023","interfaces":[{"interface":"back","average":{"1min":0.27300000000000002,"5min":0.28100000000000003,"15min":0.26400000000000001},"min":{"1min":0.19400000000000001,"5min":0.17299999999999999,"15min":0.17000000000000001},"max":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"last":0.27300000000000002},{"interface":"front","average":{"1min":0.28799999999999998,"5min":0.29099999999999998,"15min":0.27800000000000002},"min":{"1min":0.19800000000000001,"5min":0.182,"15min":0.16600000000000001},"max":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"last":0.308}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:39:54.416+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:39:54.455 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:39:54.780 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:40:09.288+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:40:09.389+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:40:10.267+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)

Ignoring malparsed backtrace: 2023-03-25T04:40:28.984 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":476,"stamp":"2023-03-25T04:40:28.245911+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002693"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":39,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:38:13.358968+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:38:13.358968+0000","last_peered":"2023-03-25T04:38:13.358968+0000","last_clean":"2023-03-25T04:38:13.358968+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:38:13.358968+0000","last_undegraded":"2023-03-25T04:38:13.358968+0000","last_fullsized":"2023-03-25T04:38:13.358968+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:38:13.358922+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:38:13.358922+0000","last_peered":"2023-03-25T04:38:13.358922+0000","last_clean":"2023-03-25T04:38:13.358922+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:38:13.358922+0000","last_undegraded":"2023-03-25T04:38:13.358922+0000","last_fullsized":"2023-03-25T04:38:13.358922+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:23.353778+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:37:23.353778+0000","last_peered":"2023-03-25T04:37:23.353778+0000","last_clean":"2023-03-25T04:37:23.353778+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:37:23.353778+0000","last_undegraded":"2023-03-25T04:37:23.353778+0000","last_fullsized":"2023-03-25T04:37:23.353778+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:40.598579+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:37:40.598579+0000","last_peered":"2023-03-25T04:37:40.598579+0000","last_clean":"2023-03-25T04:37:40.598579+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:37:40.598579+0000","last_undegraded":"2023-03-25T04:37:40.598579+0000","last_fullsized":"2023-03-25T04:37:40.598579+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":39,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:48.356344+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:37:48.356344+0000","last_peered":"2023-03-25T04:37:48.356344+0000","last_clean":"2023-03-25T04:37:48.356344+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:37:48.356344+0000","last_undegraded":"2023-03-25T04:37:48.356344+0000","last_fullsized":"2023-03-25T04:37:48.356344+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:48.356365+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:37:48.356365+0000","last_peered":"2023-03-25T04:37:48.356365+0000","last_clean":"2023-03-25T04:37:48.356365+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:37:48.356365+0000","last_undegraded":"2023-03-25T04:37:48.356365+0000","last_fullsized":"2023-03-25T04:37:48.356365+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117153,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:39:54 2023","interfaces":[{"interface":"back","average":{"1min":0.25900000000000001,"5min":0.25600000000000001,"15min":0.27100000000000002},"min":{"1min":0.188,"5min":0.17899999999999999,"15min":0.16},"max":{"1min":0.41999999999999998,"5min":0.44600000000000001,"15min":0.45100000000000001},"last":0.255},{"interface":"front","average":{"1min":0.27500000000000002,"5min":0.26600000000000001,"15min":0.27700000000000002},"min":{"1min":0.17699999999999999,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.45600000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"last":0.26800000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051743,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:39:47 2023","interfaces":[{"interface":"back","average":{"1min":0.26300000000000001,"5min":0.27700000000000002,"15min":0.26500000000000001},"min":{"1min":0.187,"5min":0.17299999999999999,"15min":0.17000000000000001},"max":{"1min":0.433,"5min":0.433,"15min":0.433},"last":0.36599999999999999},{"interface":"front","average":{"1min":0.27100000000000002,"5min":0.28799999999999998,"15min":0.27900000000000003},"min":{"1min":0.20100000000000001,"5min":0.182,"15min":0.16600000000000001},"max":{"1min":0.48599999999999999,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.34999999999999998}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:40:28.986 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean
, 2023-03-25T04:40:28.986 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}

Ignoring malparsed backtrace: 2023-03-25T04:40:47.229+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:40:47.503+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:40:48.251+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)

Ignoring malparsed backtrace: 2023-03-25T04:41:03.267 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:41:03.578 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all

Ignoring malparsed backtrace: 2023-03-25T04:41:21.347 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":502,"stamp":"2023-03-25T04:41:20.256204+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002342"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":39,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:38:13.358968+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:38:13.358968+0000","last_peered":"2023-03-25T04:38:13.358968+0000","last_clean":"2023-03-25T04:38:13.358968+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:38:13.358968+0000","last_undegraded":"2023-03-25T04:38:13.358968+0000","last_fullsized":"2023-03-25T04:38:13.358968+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:38:13.358922+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:38:13.358922+0000","last_peered":"2023-03-25T04:38:13.358922+0000","last_clean":"2023-03-25T04:38:13.358922+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:38:13.358922+0000","last_undegraded":"2023-03-25T04:38:13.358922+0000","last_fullsized":"2023-03-25T04:38:13.358922+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:23.353778+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:37:23.353778+0000","last_peered":"2023-03-25T04:37:23.353778+0000","last_clean":"2023-03-25T04:37:23.353778+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:37:23.353778+0000","last_undegraded":"2023-03-25T04:37:23.353778+0000","last_fullsized":"2023-03-25T04:37:23.353778+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:40.598579+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:37:40.598579+0000","last_peered":"2023-03-25T04:37:40.598579+0000","last_clean":"2023-03-25T04:37:40.598579+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:37:40.598579+0000","last_undegraded":"2023-03-25T04:37:40.598579+0000","last_fullsized":"2023-03-25T04:37:40.598579+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":39,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:48.356344+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:37:48.356344+0000","last_peered":"2023-03-25T04:37:48.356344+0000","last_clean":"2023-03-25T04:37:48.356344+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:37:48.356344+0000","last_undegraded":"2023-03-25T04:37:48.356344+0000","last_fullsized":"2023-03-25T04:37:48.356344+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:48.356365+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:37:48.356365+0000","last_peered":"2023-03-25T04:37:48.356365+0000","last_clean":"2023-03-25T04:37:48.356365+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:37:48.356365+0000","last_undegraded":"2023-03-25T04:37:48.356365+0000","last_fullsized":"2023-03-25T04:37:48.356365+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117163,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:40:56 2023","interfaces":[{"interface":"back","average":{"1min":0.28799999999999998,"5min":0.26300000000000001,"15min":0.27100000000000002},"min":{"1min":0.188,"5min":0.184,"15min":0.16},"max":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.17499999999999999},{"interface":"front","average":{"1min":0.29599999999999999,"5min":0.27300000000000002,"15min":0.27700000000000002},"min":{"1min":0.20599999999999999,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.43099999999999999,"5min":0.46400000000000002,"15min":0.46400000000000002},"last":0.20699999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051754,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:40:52 2023","interfaces":[{"interface":"back","average":{"1min":0.28599999999999998,"5min":0.27500000000000002,"15min":0.26700000000000002},"min":{"1min":0.17899999999999999,"5min":0.17899999999999999,"15min":0.17000000000000001},"max":{"1min":0.44,"5min":0.44,"15min":0.44},"last":0.23699999999999999},{"interface":"front","average":{"1min":0.29799999999999999,"5min":0.28699999999999998,"15min":0.28100000000000003},"min":{"1min":0.20300000000000001,"5min":0.182,"15min":0.16600000000000001},"max":{"1min":0.47599999999999998,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.28000000000000003}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:41:21.349 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean
, 2023-03-25T04:41:21.349 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}

Ignoring malparsed backtrace: 2023-03-25T04:41:39.867+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:41:40.383+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:41:40.905+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:41:58.219 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json
, 2023-03-25T04:41:58.319+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:41:58.644 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:42:18.193 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":530,"stamp":"2023-03-25T04:42:16.266659+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002356"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":39,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:38:13.358968+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:38:13.358968+0000","last_peered":"2023-03-25T04:38:13.358968+0000","last_clean":"2023-03-25T04:38:13.358968+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:38:13.358968+0000","last_undegraded":"2023-03-25T04:38:13.358968+0000","last_fullsized":"2023-03-25T04:38:13.358968+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:38:13.358922+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:38:13.358922+0000","last_peered":"2023-03-25T04:38:13.358922+0000","last_clean":"2023-03-25T04:38:13.358922+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:38:13.358922+0000","last_undegraded":"2023-03-25T04:38:13.358922+0000","last_fullsized":"2023-03-25T04:38:13.358922+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:23.364593+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:39:23.364593+0000","last_peered":"2023-03-25T04:39:23.364593+0000","last_clean":"2023-03-25T04:39:23.364593+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:39:23.364593+0000","last_undegraded":"2023-03-25T04:39:23.364593+0000","last_fullsized":"2023-03-25T04:39:23.364593+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:45.608475+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:39:45.608475+0000","last_peered":"2023-03-25T04:39:45.608475+0000","last_clean":"2023-03-25T04:39:45.608475+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:39:45.608475+0000","last_undegraded":"2023-03-25T04:39:45.608475+0000","last_fullsized":"2023-03-25T04:39:45.608475+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":40,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:53.367398+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:39:53.367398+0000","last_peered":"2023-03-25T04:39:53.367398+0000","last_clean":"2023-03-25T04:39:53.367398+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:39:53.367398+0000","last_undegraded":"2023-03-25T04:39:53.367398+0000","last_fullsized":"2023-03-25T04:39:53.367398+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:53.367424+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:39:53.367424+0000","last_peered":"2023-03-25T04:39:53.367424+0000","last_clean":"2023-03-25T04:39:53.367424+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:39:53.367424+0000","last_undegraded":"2023-03-25T04:39:53.367424+0000","last_fullsized":"2023-03-25T04:39:53.367424+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117175,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:41:58 2023","interfaces":[{"interface":"back","average":{"1min":0.25800000000000001,"5min":0.26400000000000001,"15min":0.26800000000000002},"min":{"1min":0.17499999999999999,"5min":0.17499999999999999,"15min":0.16},"max":{"1min":0.39100000000000001,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.22900000000000001},{"interface":"front","average":{"1min":0.26300000000000001,"5min":0.27200000000000002,"15min":0.27400000000000002},"min":{"1min":0.20599999999999999,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.35699999999999998,"5min":0.46400000000000002,"15min":0.46400000000000002},"last":0.26400000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051765,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:41:52 2023","interfaces":[{"interface":"back","average":{"1min":0.26500000000000001,"5min":0.27200000000000002,"15min":0.26800000000000002},"min":{"1min":0.17000000000000001,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.39200000000000002,"5min":0.44,"15min":0.44},"last":0.23899999999999999},{"interface":"front","average":{"1min":0.28100000000000003,"5min":0.28599999999999998,"15min":0.28199999999999997},"min":{"1min":0.19400000000000001,"5min":0.19400000000000001,"15min":0.16600000000000001},"max":{"1min":0.41399999999999998,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.254}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:42:18.297+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:42:18.958+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:42:35.512 INFO:teuthology.orchestra.run.smithi070.stdout:
, 2023-03-25T04:42:35.512 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":1013,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":17,"modified":"2023-03-25T04:41:54.262744+0000","services":{}},"progress_events":{}}
, 2023-03-25T04:42:35.525 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:42:52.741 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":548,"stamp":"2023-03-25T04:42:52.273601+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002221"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":40,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:40:23.370056+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:40:23.370056+0000","last_peered":"2023-03-25T04:40:23.370056+0000","last_clean":"2023-03-25T04:40:23.370056+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:40:23.370056+0000","last_undegraded":"2023-03-25T04:40:23.370056+0000","last_fullsized":"2023-03-25T04:40:23.370056+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:40:23.370039+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:40:23.370039+0000","last_peered":"2023-03-25T04:40:23.370039+0000","last_clean":"2023-03-25T04:40:23.370039+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:40:23.370039+0000","last_undegraded":"2023-03-25T04:40:23.370039+0000","last_fullsized":"2023-03-25T04:40:23.370039+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:23.364593+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:39:23.364593+0000","last_peered":"2023-03-25T04:39:23.364593+0000","last_clean":"2023-03-25T04:39:23.364593+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:39:23.364593+0000","last_undegraded":"2023-03-25T04:39:23.364593+0000","last_fullsized":"2023-03-25T04:39:23.364593+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:45.608475+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:39:45.608475+0000","last_peered":"2023-03-25T04:39:45.608475+0000","last_clean":"2023-03-25T04:39:45.608475+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:39:45.608475+0000","last_undegraded":"2023-03-25T04:39:45.608475+0000","last_fullsized":"2023-03-25T04:39:45.608475+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":40,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:53.367398+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:39:53.367398+0000","last_peered":"2023-03-25T04:39:53.367398+0000","last_clean":"2023-03-25T04:39:53.367398+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:39:53.367398+0000","last_undegraded":"2023-03-25T04:39:53.367398+0000","last_fullsized":"2023-03-25T04:39:53.367398+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:53.367424+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:39:53.367424+0000","last_peered":"2023-03-25T04:39:53.367424+0000","last_clean":"2023-03-25T04:39:53.367424+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:39:53.367424+0000","last_undegraded":"2023-03-25T04:39:53.367424+0000","last_fullsized":"2023-03-25T04:39:53.367424+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117182,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:41:58 2023","interfaces":[{"interface":"back","average":{"1min":0.25800000000000001,"5min":0.26400000000000001,"15min":0.26800000000000002},"min":{"1min":0.17499999999999999,"5min":0.17499999999999999,"15min":0.16},"max":{"1min":0.39100000000000001,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.19600000000000001},{"interface":"front","average":{"1min":0.26300000000000001,"5min":0.27200000000000002,"15min":0.27400000000000002},"min":{"1min":0.20599999999999999,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.35699999999999998,"5min":0.46400000000000002,"15min":0.46400000000000002},"last":0.219}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051773,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:41:52 2023","interfaces":[{"interface":"back","average":{"1min":0.26500000000000001,"5min":0.27200000000000002,"15min":0.26800000000000002},"min":{"1min":0.17000000000000001,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.39200000000000002,"5min":0.44,"15min":0.44},"last":0.19},{"interface":"front","average":{"1min":0.28100000000000003,"5min":0.28599999999999998,"15min":0.28199999999999997},"min":{"1min":0.19400000000000001,"5min":0.19400000000000001,"15min":0.16600000000000001},"max":{"1min":0.41399999999999998,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.17599999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:42:52.742 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean

Ignoring malparsed backtrace: 2023-03-25T04:43:09.973 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:43:09.986 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":556,"stamp":"2023-03-25T04:43:08.276556+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002359"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":40,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:40:23.370056+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:40:23.370056+0000","last_peered":"2023-03-25T04:40:23.370056+0000","last_clean":"2023-03-25T04:40:23.370056+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:40:23.370056+0000","last_undegraded":"2023-03-25T04:40:23.370056+0000","last_fullsized":"2023-03-25T04:40:23.370056+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:40:23.370039+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:40:23.370039+0000","last_peered":"2023-03-25T04:40:23.370039+0000","last_clean":"2023-03-25T04:40:23.370039+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:40:23.370039+0000","last_undegraded":"2023-03-25T04:40:23.370039+0000","last_fullsized":"2023-03-25T04:40:23.370039+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:23.364593+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:39:23.364593+0000","last_peered":"2023-03-25T04:39:23.364593+0000","last_clean":"2023-03-25T04:39:23.364593+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:39:23.364593+0000","last_undegraded":"2023-03-25T04:39:23.364593+0000","last_fullsized":"2023-03-25T04:39:23.364593+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:45.608475+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:39:45.608475+0000","last_peered":"2023-03-25T04:39:45.608475+0000","last_clean":"2023-03-25T04:39:45.608475+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:39:45.608475+0000","last_undegraded":"2023-03-25T04:39:45.608475+0000","last_fullsized":"2023-03-25T04:39:45.608475+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":40,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:53.367398+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:39:53.367398+0000","last_peered":"2023-03-25T04:39:53.367398+0000","last_clean":"2023-03-25T04:39:53.367398+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:39:53.367398+0000","last_undegraded":"2023-03-25T04:39:53.367398+0000","last_fullsized":"2023-03-25T04:39:53.367398+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:53.367424+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:39:53.367424+0000","last_peered":"2023-03-25T04:39:53.367424+0000","last_clean":"2023-03-25T04:39:53.367424+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:39:53.367424+0000","last_undegraded":"2023-03-25T04:39:53.367424+0000","last_fullsized":"2023-03-25T04:39:53.367424+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117185,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:43:00 2023","interfaces":[{"interface":"back","average":{"1min":0.27400000000000002,"5min":0.26700000000000002,"15min":0.26600000000000001},"min":{"1min":0.19600000000000001,"5min":0.17499999999999999,"15min":0.16},"max":{"1min":0.42999999999999999,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.27800000000000002},{"interface":"front","average":{"1min":0.28199999999999997,"5min":0.27500000000000002,"15min":0.27300000000000002},"min":{"1min":0.185,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.47199999999999998,"5min":0.47199999999999998,"15min":0.47199999999999998},"last":0.247}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051776,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:42:54 2023","interfaces":[{"interface":"back","average":{"1min":0.26300000000000001,"5min":0.27000000000000002,"15min":0.26900000000000002},"min":{"1min":0.17399999999999999,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.438,"5min":0.44,"15min":0.44},"last":0.20100000000000001},{"interface":"front","average":{"1min":0.27400000000000002,"5min":0.28199999999999997,"15min":0.28299999999999997},"min":{"1min":0.17599999999999999,"5min":0.17599999999999999,"15min":0.16600000000000001},"max":{"1min":0.41599999999999998,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.23499999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:43:25.042+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:43:25.411+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:43:26.013+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:43:47.937+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:43:48.534+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:43:48.889+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:44:09.318 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:44:09.331 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":586,"stamp":"2023-03-25T04:44:08.288933+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002697"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":40,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:40:23.370056+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:40:23.370056+0000","last_peered":"2023-03-25T04:40:23.370056+0000","last_clean":"2023-03-25T04:40:23.370056+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:40:23.370056+0000","last_undegraded":"2023-03-25T04:40:23.370056+0000","last_fullsized":"2023-03-25T04:40:23.370056+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:40:23.370039+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:40:23.370039+0000","last_peered":"2023-03-25T04:40:23.370039+0000","last_clean":"2023-03-25T04:40:23.370039+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:40:23.370039+0000","last_undegraded":"2023-03-25T04:40:23.370039+0000","last_fullsized":"2023-03-25T04:40:23.370039+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:23.375106+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:41:23.375106+0000","last_peered":"2023-03-25T04:41:23.375106+0000","last_clean":"2023-03-25T04:41:23.375106+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:41:23.375106+0000","last_undegraded":"2023-03-25T04:41:23.375106+0000","last_fullsized":"2023-03-25T04:41:23.375106+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:50.618930+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:41:50.618930+0000","last_peered":"2023-03-25T04:41:50.618930+0000","last_clean":"2023-03-25T04:41:50.618930+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:41:50.618930+0000","last_undegraded":"2023-03-25T04:41:50.618930+0000","last_fullsized":"2023-03-25T04:41:50.618930+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":41,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:58.377821+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:41:58.377821+0000","last_peered":"2023-03-25T04:41:58.377821+0000","last_clean":"2023-03-25T04:41:58.377821+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:41:58.377821+0000","last_undegraded":"2023-03-25T04:41:58.377821+0000","last_fullsized":"2023-03-25T04:41:58.377821+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:58.377836+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:41:58.377836+0000","last_peered":"2023-03-25T04:41:58.377836+0000","last_clean":"2023-03-25T04:41:58.377836+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:41:58.377836+0000","last_undegraded":"2023-03-25T04:41:58.377836+0000","last_fullsized":"2023-03-25T04:41:58.377836+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117197,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:44:01 2023","interfaces":[{"interface":"back","average":{"1min":0.27200000000000002,"5min":0.27000000000000002,"15min":0.26400000000000001},"min":{"1min":0.20399999999999999,"5min":0.17499999999999999,"15min":0.16},"max":{"1min":0.38400000000000001,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.245},{"interface":"front","average":{"1min":0.27600000000000002,"5min":0.27800000000000002,"15min":0.27100000000000002},"min":{"1min":0.23100000000000001,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.40400000000000003,"5min":0.47199999999999998,"15min":0.47199999999999998},"last":0.27500000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051788,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:43:55 2023","interfaces":[{"interface":"back","average":{"1min":0.26100000000000001,"5min":0.26700000000000002,"15min":0.27000000000000002},"min":{"1min":0.17799999999999999,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.374,"5min":0.44,"15min":0.44},"last":0.30299999999999999},{"interface":"front","average":{"1min":0.28299999999999997,"5min":0.28100000000000003,"15min":0.28399999999999997},"min":{"1min":0.20899999999999999,"5min":0.17599999999999999,"15min":0.16600000000000001},"max":{"1min":0.40100000000000002,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.27800000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:44:28.923 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":1126,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":19,"modified":"2023-03-25T04:43:20.278959+0000","services":{}},"progress_events":{}}
, 2023-03-25T04:44:28.936 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now
, 2023-03-25T04:44:28.937 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace: 2023-03-25T04:44:48.786 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:44:48.800 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":606,"stamp":"2023-03-25T04:44:48.296805+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002182"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":41,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:42:34.382196+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:42:34.382196+0000","last_peered":"2023-03-25T04:42:34.382196+0000","last_clean":"2023-03-25T04:42:34.382196+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:42:34.382196+0000","last_undegraded":"2023-03-25T04:42:34.382196+0000","last_fullsized":"2023-03-25T04:42:34.382196+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:42:29.381744+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:42:29.381744+0000","last_peered":"2023-03-25T04:42:29.381744+0000","last_clean":"2023-03-25T04:42:29.381744+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:42:29.381744+0000","last_undegraded":"2023-03-25T04:42:29.381744+0000","last_fullsized":"2023-03-25T04:42:29.381744+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:23.375106+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:41:23.375106+0000","last_peered":"2023-03-25T04:41:23.375106+0000","last_clean":"2023-03-25T04:41:23.375106+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:41:23.375106+0000","last_undegraded":"2023-03-25T04:41:23.375106+0000","last_fullsized":"2023-03-25T04:41:23.375106+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:50.618930+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:41:50.618930+0000","last_peered":"2023-03-25T04:41:50.618930+0000","last_clean":"2023-03-25T04:41:50.618930+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:41:50.618930+0000","last_undegraded":"2023-03-25T04:41:50.618930+0000","last_fullsized":"2023-03-25T04:41:50.618930+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":41,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:58.377821+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:41:58.377821+0000","last_peered":"2023-03-25T04:41:58.377821+0000","last_clean":"2023-03-25T04:41:58.377821+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:41:58.377821+0000","last_undegraded":"2023-03-25T04:41:58.377821+0000","last_fullsized":"2023-03-25T04:41:58.377821+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:58.377836+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:41:58.377836+0000","last_peered":"2023-03-25T04:41:58.377836+0000","last_clean":"2023-03-25T04:41:58.377836+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:41:58.377836+0000","last_undegraded":"2023-03-25T04:41:58.377836+0000","last_fullsized":"2023-03-25T04:41:58.377836+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117206,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:44:01 2023","interfaces":[{"interface":"back","average":{"1min":0.27200000000000002,"5min":0.27000000000000002,"15min":0.26400000000000001},"min":{"1min":0.20399999999999999,"5min":0.17499999999999999,"15min":0.16},"max":{"1min":0.38400000000000001,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.216},{"interface":"front","average":{"1min":0.27600000000000002,"5min":0.27800000000000002,"15min":0.27100000000000002},"min":{"1min":0.23100000000000001,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.40400000000000003,"5min":0.47199999999999998,"15min":0.47199999999999998},"last":0.23999999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051796,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:43:55 2023","interfaces":[{"interface":"back","average":{"1min":0.26100000000000001,"5min":0.26700000000000002,"15min":0.27000000000000002},"min":{"1min":0.17799999999999999,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.374,"5min":0.44,"15min":0.44},"last":0.25600000000000001},{"interface":"front","average":{"1min":0.28299999999999997,"5min":0.28100000000000003,"15min":0.28399999999999997},"min":{"1min":0.20899999999999999,"5min":0.17599999999999999,"15min":0.16600000000000001},"max":{"1min":0.40100000000000002,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.28000000000000003}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:45:05.762 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:45:06.073 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all

Ignoring malparsed backtrace: 2023-03-25T04:45:25.658 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":624,"stamp":"2023-03-25T04:45:24.304294+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002366"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":41,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:42:34.382196+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:42:34.382196+0000","last_peered":"2023-03-25T04:42:34.382196+0000","last_clean":"2023-03-25T04:42:34.382196+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:42:34.382196+0000","last_undegraded":"2023-03-25T04:42:34.382196+0000","last_fullsized":"2023-03-25T04:42:34.382196+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:42:29.381744+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:42:29.381744+0000","last_peered":"2023-03-25T04:42:29.381744+0000","last_clean":"2023-03-25T04:42:29.381744+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:42:29.381744+0000","last_undegraded":"2023-03-25T04:42:29.381744+0000","last_fullsized":"2023-03-25T04:42:29.381744+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:23.375106+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:41:23.375106+0000","last_peered":"2023-03-25T04:41:23.375106+0000","last_clean":"2023-03-25T04:41:23.375106+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:41:23.375106+0000","last_undegraded":"2023-03-25T04:41:23.375106+0000","last_fullsized":"2023-03-25T04:41:23.375106+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:50.618930+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:41:50.618930+0000","last_peered":"2023-03-25T04:41:50.618930+0000","last_clean":"2023-03-25T04:41:50.618930+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:41:50.618930+0000","last_undegraded":"2023-03-25T04:41:50.618930+0000","last_fullsized":"2023-03-25T04:41:50.618930+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":41,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:58.377821+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:41:58.377821+0000","last_peered":"2023-03-25T04:41:58.377821+0000","last_clean":"2023-03-25T04:41:58.377821+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:41:58.377821+0000","last_undegraded":"2023-03-25T04:41:58.377821+0000","last_fullsized":"2023-03-25T04:41:58.377821+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:58.377836+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:41:58.377836+0000","last_peered":"2023-03-25T04:41:58.377836+0000","last_clean":"2023-03-25T04:41:58.377836+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:41:58.377836+0000","last_undegraded":"2023-03-25T04:41:58.377836+0000","last_fullsized":"2023-03-25T04:41:58.377836+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117213,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:45:01 2023","interfaces":[{"interface":"back","average":{"1min":0.251,"5min":0.26800000000000002,"15min":0.26400000000000001},"min":{"1min":0.183,"5min":0.17499999999999999,"15min":0.16},"max":{"1min":0.41499999999999998,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.26900000000000002},{"interface":"front","average":{"1min":0.28000000000000003,"5min":0.27900000000000003,"15min":0.27100000000000002},"min":{"1min":0.20999999999999999,"5min":0.185,"15min":0.17499999999999999},"max":{"1min":0.44400000000000001,"5min":0.47199999999999998,"15min":0.47199999999999998},"last":0.27900000000000003}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051803,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:44:55 2023","interfaces":[{"interface":"back","average":{"1min":0.252,"5min":0.26500000000000001,"15min":0.26900000000000002},"min":{"1min":0.183,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.39700000000000002,"5min":0.44,"15min":0.44},"last":0.20699999999999999},{"interface":"front","average":{"1min":0.27000000000000002,"5min":0.28100000000000003,"15min":0.28299999999999997},"min":{"1min":0.20300000000000001,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":0.42099999999999999,"5min":0.47599999999999998,"15min":0.48599999999999999},"last":0.23599999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:45:25.659 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:45:25.969 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:45:42.735+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:45:42.744+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:45:42.897 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace: 2023-03-25T04:46:00.183 INFO:teuthology.orchestra.run.smithi070.stdout:
, 2023-03-25T04:46:00.183 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":1217,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":20,"modified":"2023-03-25T04:45:42.307700+0000","services":{"mgr":{"daemons":{"summary":"","x":{"start_epoch":0,"start_stamp":"0.000000","gid":0,"addr":"(unrecognized address family 0)/0","metadata":{},"task_status":{}}}}}},"progress_events":{}}
, 2023-03-25T04:46:00.197 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now

Ignoring malparsed backtrace: 2023-03-25T04:46:19.910+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:46:20.039 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all

Ignoring malparsed backtrace: 2023-03-25T04:46:39.611 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}
, 2023-03-25T04:46:39.611 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json
, 2023-03-25T04:46:39.613+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:46:58.485+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:46:58.764+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:46:59.519+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:47:14.132 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:47:14.146 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":678,"stamp":"2023-03-25T04:47:12.325044+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002571"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":42,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:44.393426+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:44:44.393426+0000","last_peered":"2023-03-25T04:44:44.393426+0000","last_clean":"2023-03-25T04:44:44.393426+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:44:44.393426+0000","last_undegraded":"2023-03-25T04:44:44.393426+0000","last_fullsized":"2023-03-25T04:44:44.393426+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:39.393044+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:44:39.393044+0000","last_peered":"2023-03-25T04:44:39.393044+0000","last_clean":"2023-03-25T04:44:39.393044+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:44:39.393044+0000","last_undegraded":"2023-03-25T04:44:39.393044+0000","last_fullsized":"2023-03-25T04:44:39.393044+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:43:24.386177+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:43:24.386177+0000","last_peered":"2023-03-25T04:43:24.386177+0000","last_clean":"2023-03-25T04:43:24.386177+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:43:24.386177+0000","last_undegraded":"2023-03-25T04:43:24.386177+0000","last_fullsized":"2023-03-25T04:43:24.386177+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:43:55.629798+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:43:55.629798+0000","last_peered":"2023-03-25T04:43:55.629798+0000","last_clean":"2023-03-25T04:43:55.629798+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:43:55.629798+0000","last_undegraded":"2023-03-25T04:43:55.629798+0000","last_fullsized":"2023-03-25T04:43:55.629798+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":42,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:04.389904+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:44:04.389904+0000","last_peered":"2023-03-25T04:44:04.389904+0000","last_clean":"2023-03-25T04:44:04.389904+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:44:04.389904+0000","last_undegraded":"2023-03-25T04:44:04.389904+0000","last_fullsized":"2023-03-25T04:44:04.389904+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:04.389923+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:44:04.389923+0000","last_peered":"2023-03-25T04:44:04.389923+0000","last_clean":"2023-03-25T04:44:04.389923+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:44:04.389923+0000","last_undegraded":"2023-03-25T04:44:04.389923+0000","last_fullsized":"2023-03-25T04:44:04.389923+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117235,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:47:05 2023","interfaces":[{"interface":"back","average":{"1min":0.27800000000000002,"5min":0.27000000000000002,"15min":0.26300000000000001},"min":{"1min":0.20100000000000001,"5min":0.16900000000000001,"15min":0.16},"max":{"1min":0.39200000000000002,"5min":0.42999999999999999,"15min":0.45600000000000002},"last":0.379},{"interface":"front","average":{"1min":0.28399999999999997,"5min":0.28100000000000003,"15min":0.27200000000000002},"min":{"1min":0.17799999999999999,"5min":0.17799999999999999,"15min":0.17699999999999999},"max":{"1min":0.42599999999999999,"5min":0.47199999999999998,"15min":0.47199999999999998},"last":0.39800000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051825,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:47:01 2023","interfaces":[{"interface":"back","average":{"1min":0.27300000000000002,"5min":0.26100000000000001,"15min":0.27000000000000002},"min":{"1min":0.186,"5min":0.17399999999999999,"15min":0.17000000000000001},"max":{"1min":0.40400000000000003,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.23899999999999999},{"interface":"front","average":{"1min":0.28699999999999998,"5min":0.27500000000000002,"15min":0.28299999999999997},"min":{"1min":0.186,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.48599999999999999},"last":0.19600000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:47:31.198 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json
, 2023-03-25T04:47:31.411+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:47:31.789 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:47:49.509 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":696,"stamp":"2023-03-25T04:47:48.332093+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002697"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":42,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:44.393426+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:44:44.393426+0000","last_peered":"2023-03-25T04:44:44.393426+0000","last_clean":"2023-03-25T04:44:44.393426+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:44:44.393426+0000","last_undegraded":"2023-03-25T04:44:44.393426+0000","last_fullsized":"2023-03-25T04:44:44.393426+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:39.393044+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:44:39.393044+0000","last_peered":"2023-03-25T04:44:39.393044+0000","last_clean":"2023-03-25T04:44:39.393044+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:44:39.393044+0000","last_undegraded":"2023-03-25T04:44:39.393044+0000","last_fullsized":"2023-03-25T04:44:39.393044+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:45:24.396717+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:45:24.396717+0000","last_peered":"2023-03-25T04:45:24.396717+0000","last_clean":"2023-03-25T04:45:24.396717+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:45:24.396717+0000","last_undegraded":"2023-03-25T04:45:24.396717+0000","last_fullsized":"2023-03-25T04:45:24.396717+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:43:55.629798+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:43:55.629798+0000","last_peered":"2023-03-25T04:43:55.629798+0000","last_clean":"2023-03-25T04:43:55.629798+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:43:55.629798+0000","last_undegraded":"2023-03-25T04:43:55.629798+0000","last_fullsized":"2023-03-25T04:43:55.629798+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":42,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:04.389904+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:44:04.389904+0000","last_peered":"2023-03-25T04:44:04.389904+0000","last_clean":"2023-03-25T04:44:04.389904+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:44:04.389904+0000","last_undegraded":"2023-03-25T04:44:04.389904+0000","last_fullsized":"2023-03-25T04:44:04.389904+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:04.389923+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:44:04.389923+0000","last_peered":"2023-03-25T04:44:04.389923+0000","last_clean":"2023-03-25T04:44:04.389923+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:44:04.389923+0000","last_undegraded":"2023-03-25T04:44:04.389923+0000","last_fullsized":"2023-03-25T04:44:04.389923+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117242,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:47:05 2023","interfaces":[{"interface":"back","average":{"1min":0.27800000000000002,"5min":0.27000000000000002,"15min":0.26300000000000001},"min":{"1min":0.20100000000000001,"5min":0.16900000000000001,"15min":0.16},"max":{"1min":0.39200000000000002,"5min":0.42999999999999999,"15min":0.45600000000000002},"last":0.40500000000000003},{"interface":"front","average":{"1min":0.28399999999999997,"5min":0.28100000000000003,"15min":0.27200000000000002},"min":{"1min":0.17799999999999999,"5min":0.17799999999999999,"15min":0.17699999999999999},"max":{"1min":0.42599999999999999,"5min":0.47199999999999998,"15min":0.47199999999999998},"last":0.42399999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051834,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:47:01 2023","interfaces":[{"interface":"back","average":{"1min":0.27300000000000002,"5min":0.26100000000000001,"15min":0.27000000000000002},"min":{"1min":0.186,"5min":0.17399999999999999,"15min":0.17000000000000001},"max":{"1min":0.40400000000000003,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.25600000000000001},{"interface":"front","average":{"1min":0.28699999999999998,"5min":0.27500000000000002,"15min":0.28299999999999997},"min":{"1min":0.186,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.48599999999999999},"last":0.22800000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:47:49.510 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace:     ctx.managers[config['cluster']].wait_for_clean()
,   File "/home/teuthworker/src/github.com_ceph_ceph-c_511015c23eef0684a64171b710e432448df2b7c7/qa/tasks/ceph_manager.py", line 2747, in wait_for_clean
,     assert time.time() - start < timeout, \

Ignoring malparsed backtrace:  ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
,  1: /lib64/libpthread.so.0(+0x12cf0) [0x7f1453c2fcf0]
,  2: __pthread_mutex_lock()

Ignoring malparsed backtrace: 2023-03-25T04:28:05.172 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":142,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":5,"modified":"2023-03-25T04:27:12.092098+0000","services":{}},"progress_events":{}}
, 2023-03-25T04:28:05.185 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now
, 2023-03-25T04:28:05.186 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace: 2023-03-25T04:28:28.360 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":116,"stamp":"2023-03-25T04:28:28.108007+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002578"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.314661+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:27:45.314661+0000","last_peered":"2023-03-25T04:27:45.314661+0000","last_clean":"2023-03-25T04:27:45.314661+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:27:45.314661+0000","last_undegraded":"2023-03-25T04:27:45.314661+0000","last_fullsized":"2023-03-25T04:27:45.314661+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":25,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.310745+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:27:44.310745+0000","last_peered":"2023-03-25T04:27:44.310745+0000","last_clean":"2023-03-25T04:27:44.310745+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:27:44.310745+0000","last_undegraded":"2023-03-25T04:27:44.310745+0000","last_fullsized":"2023-03-25T04:27:44.310745+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":25,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.310650+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:27:44.310650+0000","last_peered":"2023-03-25T04:27:44.310650+0000","last_clean":"2023-03-25T04:27:44.310650+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:27:44.310650+0000","last_undegraded":"2023-03-25T04:27:44.310650+0000","last_fullsized":"2023-03-25T04:27:44.310650+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":26,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.312962+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:27:44.312962+0000","last_peered":"2023-03-25T04:27:44.312962+0000","last_clean":"2023-03-25T04:27:44.312962+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:27:44.312962+0000","last_undegraded":"2023-03-25T04:27:44.312962+0000","last_fullsized":"2023-03-25T04:27:44.312962+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315064+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:27:45.315064+0000","last_peered":"2023-03-25T04:27:45.315064+0000","last_clean":"2023-03-25T04:27:45.315064+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:27:45.315064+0000","last_undegraded":"2023-03-25T04:27:45.315064+0000","last_fullsized":"2023-03-25T04:27:45.315064+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":25,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.310707+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:27:44.310707+0000","last_peered":"2023-03-25T04:27:44.310707+0000","last_clean":"2023-03-25T04:27:44.310707+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:27:44.310707+0000","last_undegraded":"2023-03-25T04:27:44.310707+0000","last_fullsized":"2023-03-25T04:27:44.310707+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117007,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.375}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051597,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.186}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:28:28.362 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean
, 2023-03-25T04:28:28.362 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:28:45.655 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":124,"stamp":"2023-03-25T04:28:44.111265+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002492"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.314661+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:27:45.314661+0000","last_peered":"2023-03-25T04:27:45.314661+0000","last_clean":"2023-03-25T04:27:45.314661+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:27:45.314661+0000","last_undegraded":"2023-03-25T04:27:45.314661+0000","last_fullsized":"2023-03-25T04:27:45.314661+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":25,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.310745+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:27:44.310745+0000","last_peered":"2023-03-25T04:27:44.310745+0000","last_clean":"2023-03-25T04:27:44.310745+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:27:44.310745+0000","last_undegraded":"2023-03-25T04:27:44.310745+0000","last_fullsized":"2023-03-25T04:27:44.310745+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":25,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.310650+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:27:44.310650+0000","last_peered":"2023-03-25T04:27:44.310650+0000","last_clean":"2023-03-25T04:27:44.310650+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:27:44.310650+0000","last_undegraded":"2023-03-25T04:27:44.310650+0000","last_fullsized":"2023-03-25T04:27:44.310650+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":26,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.312962+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:27:44.312962+0000","last_peered":"2023-03-25T04:27:44.312962+0000","last_clean":"2023-03-25T04:27:44.312962+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:27:44.312962+0000","last_undegraded":"2023-03-25T04:27:44.312962+0000","last_fullsized":"2023-03-25T04:27:44.312962+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315064+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:27:45.315064+0000","last_peered":"2023-03-25T04:27:45.315064+0000","last_clean":"2023-03-25T04:27:45.315064+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:27:45.315064+0000","last_undegraded":"2023-03-25T04:27:45.315064+0000","last_fullsized":"2023-03-25T04:27:45.315064+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":25,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.310707+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:27:44.310707+0000","last_peered":"2023-03-25T04:27:44.310707+0000","last_clean":"2023-03-25T04:27:44.310707+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:27:44.310707+0000","last_undegraded":"2023-03-25T04:27:44.310707+0000","last_fullsized":"2023-03-25T04:27:44.310707+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117011,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:28:31 2023","interfaces":[{"interface":"back","average":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.36899999999999999},{"interface":"front","average":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"min":{"1min":0.19,"5min":0.19,"15min":0.19},"max":{"1min":0.46300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.39800000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051601,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:28:24 2023","interfaces":[{"interface":"back","average":{"1min":0.249,"5min":0.249,"15min":0.249},"min":{"1min":0.17699999999999999,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"last":0.254},{"interface":"front","average":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"min":{"1min":0.20200000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"last":0.312}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:28:45.656 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace: 2023-03-25T04:29:05.171 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}
, 2023-03-25T04:29:05.171 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json
, 2023-03-25T04:29:05.561 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:29:22.465 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:29:22.773 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all

Ignoring malparsed backtrace: 2023-03-25T04:29:42.405 INFO:teuthology.orchestra.run.smithi070.stdout:
, 2023-03-25T04:29:42.406 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":240,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":9,"modified":"2023-03-25T04:29:32.120358+0000","services":{"mon":{"daemons":{"summary":"","a":{"start_epoch":0,"start_stamp":"0.000000","gid":0,"addr":"(unrecognized address family 0)/0","metadata":{},"task_status":{}}}}}},"progress_events":{}}
, 2023-03-25T04:29:42.419 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:29:59.627 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":161,"stamp":"2023-03-25T04:29:58.125186+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002398"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315468+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:27:45.315468+0000","last_peered":"2023-03-25T04:27:45.315468+0000","last_clean":"2023-03-25T04:27:45.315468+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:27:45.315468+0000","last_undegraded":"2023-03-25T04:27:45.315468+0000","last_fullsized":"2023-03-25T04:27:45.315468+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312845+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:27:45.312845+0000","last_peered":"2023-03-25T04:27:45.312845+0000","last_clean":"2023-03-25T04:27:45.312845+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:27:45.312845+0000","last_undegraded":"2023-03-25T04:27:45.312845+0000","last_fullsized":"2023-03-25T04:27:45.312845+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312882+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:27:45.312882+0000","last_peered":"2023-03-25T04:27:45.312882+0000","last_clean":"2023-03-25T04:27:45.312882+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:27:45.312882+0000","last_undegraded":"2023-03-25T04:27:45.312882+0000","last_fullsized":"2023-03-25T04:27:45.312882+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.311730+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:27:45.311730+0000","last_peered":"2023-03-25T04:27:45.311730+0000","last_clean":"2023-03-25T04:27:45.311730+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:27:45.311730+0000","last_undegraded":"2023-03-25T04:27:45.311730+0000","last_fullsized":"2023-03-25T04:27:45.311730+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315735+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:27:45.315735+0000","last_peered":"2023-03-25T04:27:45.315735+0000","last_clean":"2023-03-25T04:27:45.315735+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:27:45.315735+0000","last_undegraded":"2023-03-25T04:27:45.315735+0000","last_fullsized":"2023-03-25T04:27:45.315735+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312833+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:27:45.312833+0000","last_peered":"2023-03-25T04:27:45.312833+0000","last_clean":"2023-03-25T04:27:45.312833+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:27:45.312833+0000","last_undegraded":"2023-03-25T04:27:45.312833+0000","last_fullsized":"2023-03-25T04:27:45.312833+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117026,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:29:33 2023","interfaces":[{"interface":"back","average":{"1min":0.26300000000000001,"5min":0.29099999999999998,"15min":0.29599999999999999},"min":{"1min":0.191,"5min":0.191,"15min":0.191},"max":{"1min":0.371,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.20499999999999999},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.29599999999999999,"15min":0.29999999999999999},"min":{"1min":0.21099999999999999,"5min":0.19,"15min":0.19},"max":{"1min":0.39800000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.19500000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051616,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:29:29 2023","interfaces":[{"interface":"back","average":{"1min":0.27200000000000002,"5min":0.253,"15min":0.25},"min":{"1min":0.183,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"last":0.26400000000000001},{"interface":"front","average":{"1min":0.28499999999999998,"5min":0.26800000000000002,"15min":0.26500000000000001},"min":{"1min":0.16600000000000001,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"last":0.29899999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:29:59.629 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean

Ignoring malparsed backtrace: 2023-03-25T04:30:16.889 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:30:16.903 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":170,"stamp":"2023-03-25T04:30:16.128427+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002222"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315468+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:27:45.315468+0000","last_peered":"2023-03-25T04:27:45.315468+0000","last_clean":"2023-03-25T04:27:45.315468+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:27:45.315468+0000","last_undegraded":"2023-03-25T04:27:45.315468+0000","last_fullsized":"2023-03-25T04:27:45.315468+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312845+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:27:45.312845+0000","last_peered":"2023-03-25T04:27:45.312845+0000","last_clean":"2023-03-25T04:27:45.312845+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:27:45.312845+0000","last_undegraded":"2023-03-25T04:27:45.312845+0000","last_fullsized":"2023-03-25T04:27:45.312845+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312882+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:27:45.312882+0000","last_peered":"2023-03-25T04:27:45.312882+0000","last_clean":"2023-03-25T04:27:45.312882+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:27:45.312882+0000","last_undegraded":"2023-03-25T04:27:45.312882+0000","last_fullsized":"2023-03-25T04:27:45.312882+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.311730+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:27:45.311730+0000","last_peered":"2023-03-25T04:27:45.311730+0000","last_clean":"2023-03-25T04:27:45.311730+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:27:45.311730+0000","last_undegraded":"2023-03-25T04:27:45.311730+0000","last_fullsized":"2023-03-25T04:27:45.311730+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315735+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:27:45.315735+0000","last_peered":"2023-03-25T04:27:45.315735+0000","last_clean":"2023-03-25T04:27:45.315735+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:27:45.315735+0000","last_undegraded":"2023-03-25T04:27:45.315735+0000","last_fullsized":"2023-03-25T04:27:45.315735+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312833+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:27:45.312833+0000","last_peered":"2023-03-25T04:27:45.312833+0000","last_clean":"2023-03-25T04:27:45.312833+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:27:45.312833+0000","last_undegraded":"2023-03-25T04:27:45.312833+0000","last_fullsized":"2023-03-25T04:27:45.312833+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117030,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:29:33 2023","interfaces":[{"interface":"back","average":{"1min":0.26300000000000001,"5min":0.29099999999999998,"15min":0.29599999999999999},"min":{"1min":0.191,"5min":0.191,"15min":0.191},"max":{"1min":0.371,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.245},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.29599999999999999,"15min":0.29999999999999999},"min":{"1min":0.21099999999999999,"5min":0.19,"15min":0.19},"max":{"1min":0.39800000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.20899999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051620,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:29:29 2023","interfaces":[{"interface":"back","average":{"1min":0.27200000000000002,"5min":0.253,"15min":0.25},"min":{"1min":0.183,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"last":0.19400000000000001},{"interface":"front","average":{"1min":0.28499999999999998,"5min":0.26800000000000002,"15min":0.26500000000000001},"min":{"1min":0.16600000000000001,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"last":0.23599999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:30:31.984+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:30:32.201+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:30:32.981+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)

Ignoring malparsed backtrace: 2023-03-25T04:30:48.449 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now
, 2023-03-25T04:30:48.450 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:30:48.760 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:31:08.342 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":196,"stamp":"2023-03-25T04:31:08.139018+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002431"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315468+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:27:45.315468+0000","last_peered":"2023-03-25T04:27:45.315468+0000","last_clean":"2023-03-25T04:27:45.315468+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:27:45.315468+0000","last_undegraded":"2023-03-25T04:27:45.315468+0000","last_fullsized":"2023-03-25T04:27:45.315468+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312845+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:27:45.312845+0000","last_peered":"2023-03-25T04:27:45.312845+0000","last_clean":"2023-03-25T04:27:45.312845+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:27:45.312845+0000","last_undegraded":"2023-03-25T04:27:45.312845+0000","last_fullsized":"2023-03-25T04:27:45.312845+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312882+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:27:45.312882+0000","last_peered":"2023-03-25T04:27:45.312882+0000","last_clean":"2023-03-25T04:27:45.312882+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:27:45.312882+0000","last_undegraded":"2023-03-25T04:27:45.312882+0000","last_fullsized":"2023-03-25T04:27:45.312882+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.311730+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:27:45.311730+0000","last_peered":"2023-03-25T04:27:45.311730+0000","last_clean":"2023-03-25T04:27:45.311730+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:27:45.311730+0000","last_undegraded":"2023-03-25T04:27:45.311730+0000","last_fullsized":"2023-03-25T04:27:45.311730+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315735+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:27:45.315735+0000","last_peered":"2023-03-25T04:27:45.315735+0000","last_clean":"2023-03-25T04:27:45.315735+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:27:45.315735+0000","last_undegraded":"2023-03-25T04:27:45.315735+0000","last_fullsized":"2023-03-25T04:27:45.315735+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312833+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:27:45.312833+0000","last_peered":"2023-03-25T04:27:45.312833+0000","last_clean":"2023-03-25T04:27:45.312833+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:27:45.312833+0000","last_undegraded":"2023-03-25T04:27:45.312833+0000","last_fullsized":"2023-03-25T04:27:45.312833+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117040,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:30:37 2023","interfaces":[{"interface":"back","average":{"1min":0.28799999999999998,"5min":0.28899999999999998,"15min":0.29499999999999998},"min":{"1min":0.20200000000000001,"5min":0.191,"15min":0.191},"max":{"1min":0.433,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.29799999999999999},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.28999999999999998,"15min":0.29799999999999999},"min":{"1min":0.17499999999999999,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":0.40300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.25700000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051630,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:30:32 2023","interfaces":[{"interface":"back","average":{"1min":0.25600000000000001,"5min":0.255,"15min":0.251},"min":{"1min":0.19400000000000001,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.35699999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"last":0.22800000000000001},{"interface":"front","average":{"1min":0.28699999999999998,"5min":0.27200000000000002,"15min":0.26600000000000001},"min":{"1min":0.20599999999999999,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.39400000000000002,"5min":0.42899999999999999,"15min":0.42899999999999999},"last":0.24099999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:31:08.343 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace: 2023-03-25T04:31:24.283+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:31:25.192+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:31:25.295+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)

Ignoring malparsed backtrace: 2023-03-25T04:31:40.257 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:31:40.271 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":212,"stamp":"2023-03-25T04:31:40.144732+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002208"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315468+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:27:45.315468+0000","last_peered":"2023-03-25T04:27:45.315468+0000","last_clean":"2023-03-25T04:27:45.315468+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:27:45.315468+0000","last_undegraded":"2023-03-25T04:27:45.315468+0000","last_fullsized":"2023-03-25T04:27:45.315468+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312845+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:27:45.312845+0000","last_peered":"2023-03-25T04:27:45.312845+0000","last_clean":"2023-03-25T04:27:45.312845+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:27:45.312845+0000","last_undegraded":"2023-03-25T04:27:45.312845+0000","last_fullsized":"2023-03-25T04:27:45.312845+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:23.308607+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:29:23.308607+0000","last_peered":"2023-03-25T04:29:23.308607+0000","last_clean":"2023-03-25T04:29:23.308607+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:29:23.308607+0000","last_undegraded":"2023-03-25T04:29:23.308607+0000","last_fullsized":"2023-03-25T04:29:23.308607+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":28,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:20.554710+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:29:20.554710+0000","last_peered":"2023-03-25T04:29:20.554710+0000","last_clean":"2023-03-25T04:29:20.554710+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:29:20.554710+0000","last_undegraded":"2023-03-25T04:29:20.554710+0000","last_fullsized":"2023-03-25T04:29:20.554710+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309216+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:29:28.309216+0000","last_peered":"2023-03-25T04:29:28.309216+0000","last_clean":"2023-03-25T04:29:28.309216+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:29:28.309216+0000","last_undegraded":"2023-03-25T04:29:28.309216+0000","last_fullsized":"2023-03-25T04:29:28.309216+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309233+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:29:28.309233+0000","last_peered":"2023-03-25T04:29:28.309233+0000","last_clean":"2023-03-25T04:29:28.309233+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:29:28.309233+0000","last_undegraded":"2023-03-25T04:29:28.309233+0000","last_fullsized":"2023-03-25T04:29:28.309233+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117046,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:30:37 2023","interfaces":[{"interface":"back","average":{"1min":0.28799999999999998,"5min":0.28899999999999998,"15min":0.29499999999999998},"min":{"1min":0.20200000000000001,"5min":0.191,"15min":0.191},"max":{"1min":0.433,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.216},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.28999999999999998,"15min":0.29799999999999999},"min":{"1min":0.17499999999999999,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":0.40300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.24299999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051637,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:31:35 2023","interfaces":[{"interface":"back","average":{"1min":0.26100000000000001,"5min":0.25700000000000001,"15min":0.251},"min":{"1min":0.19,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.35899999999999999,"5min":0.41599999999999998,"15min":0.41599999999999998},"last":0.221},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.27400000000000002,"15min":0.26700000000000002},"min":{"1min":0.20200000000000001,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.374,"5min":0.42899999999999999,"15min":0.42899999999999999},"last":0.24099999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:32:01.906+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:32:02.388+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:32:02.794 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:32:22.326 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":233,"stamp":"2023-03-25T04:32:22.153289+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002545"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:33.309748+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:29:33.309748+0000","last_peered":"2023-03-25T04:29:33.309748+0000","last_clean":"2023-03-25T04:29:33.309748+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:29:33.309748+0000","last_undegraded":"2023-03-25T04:29:33.309748+0000","last_fullsized":"2023-03-25T04:29:33.309748+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:33.309729+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:29:33.309729+0000","last_peered":"2023-03-25T04:29:33.309729+0000","last_clean":"2023-03-25T04:29:33.309729+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:29:33.309729+0000","last_undegraded":"2023-03-25T04:29:33.309729+0000","last_fullsized":"2023-03-25T04:29:33.309729+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:23.308607+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:29:23.308607+0000","last_peered":"2023-03-25T04:29:23.308607+0000","last_clean":"2023-03-25T04:29:23.308607+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:29:23.308607+0000","last_undegraded":"2023-03-25T04:29:23.308607+0000","last_fullsized":"2023-03-25T04:29:23.308607+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":28,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:20.554710+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:29:20.554710+0000","last_peered":"2023-03-25T04:29:20.554710+0000","last_clean":"2023-03-25T04:29:20.554710+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:29:20.554710+0000","last_undegraded":"2023-03-25T04:29:20.554710+0000","last_fullsized":"2023-03-25T04:29:20.554710+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309216+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:29:28.309216+0000","last_peered":"2023-03-25T04:29:28.309216+0000","last_clean":"2023-03-25T04:29:28.309216+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:29:28.309216+0000","last_undegraded":"2023-03-25T04:29:28.309216+0000","last_fullsized":"2023-03-25T04:29:28.309216+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309233+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:29:28.309233+0000","last_peered":"2023-03-25T04:29:28.309233+0000","last_clean":"2023-03-25T04:29:28.309233+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:29:28.309233+0000","last_undegraded":"2023-03-25T04:29:28.309233+0000","last_fullsized":"2023-03-25T04:29:28.309233+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117055,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:31:42 2023","interfaces":[{"interface":"back","average":{"1min":0.27500000000000002,"5min":0.28399999999999997,"15min":0.29399999999999998},"min":{"1min":0.19800000000000001,"5min":0.191,"15min":0.191},"max":{"1min":0.36899999999999999,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.32300000000000001},{"interface":"front","average":{"1min":0.27900000000000003,"5min":0.28499999999999998,"15min":0.29599999999999999},"min":{"1min":0.216,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":0.40799999999999997,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.27100000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051646,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:31:35 2023","interfaces":[{"interface":"back","average":{"1min":0.26100000000000001,"5min":0.25700000000000001,"15min":0.251},"min":{"1min":0.19,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.35899999999999999,"5min":0.41599999999999998,"15min":0.41599999999999998},"last":0.25600000000000001},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.27400000000000002,"15min":0.26700000000000002},"min":{"1min":0.20200000000000001,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.374,"5min":0.42899999999999999,"15min":0.42899999999999999},"last":0.29199999999999998}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:32:22.327 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean
, 2023-03-25T04:32:22.327 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}

Ignoring malparsed backtrace: 2023-03-25T04:32:39.836 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:32:39.849 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":241,"stamp":"2023-03-25T04:32:38.156312+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002346"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:33.309748+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:29:33.309748+0000","last_peered":"2023-03-25T04:29:33.309748+0000","last_clean":"2023-03-25T04:29:33.309748+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:29:33.309748+0000","last_undegraded":"2023-03-25T04:29:33.309748+0000","last_fullsized":"2023-03-25T04:29:33.309748+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:33.309729+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:29:33.309729+0000","last_peered":"2023-03-25T04:29:33.309729+0000","last_clean":"2023-03-25T04:29:33.309729+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:29:33.309729+0000","last_undegraded":"2023-03-25T04:29:33.309729+0000","last_fullsized":"2023-03-25T04:29:33.309729+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:23.308607+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:29:23.308607+0000","last_peered":"2023-03-25T04:29:23.308607+0000","last_clean":"2023-03-25T04:29:23.308607+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:29:23.308607+0000","last_undegraded":"2023-03-25T04:29:23.308607+0000","last_fullsized":"2023-03-25T04:29:23.308607+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":28,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:20.554710+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:29:20.554710+0000","last_peered":"2023-03-25T04:29:20.554710+0000","last_clean":"2023-03-25T04:29:20.554710+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:29:20.554710+0000","last_undegraded":"2023-03-25T04:29:20.554710+0000","last_fullsized":"2023-03-25T04:29:20.554710+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309216+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:29:28.309216+0000","last_peered":"2023-03-25T04:29:28.309216+0000","last_clean":"2023-03-25T04:29:28.309216+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:29:28.309216+0000","last_undegraded":"2023-03-25T04:29:28.309216+0000","last_fullsized":"2023-03-25T04:29:28.309216+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309233+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:29:28.309233+0000","last_peered":"2023-03-25T04:29:28.309233+0000","last_clean":"2023-03-25T04:29:28.309233+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:29:28.309233+0000","last_undegraded":"2023-03-25T04:29:28.309233+0000","last_fullsized":"2023-03-25T04:29:28.309233+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117058,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:31:42 2023","interfaces":[{"interface":"back","average":{"1min":0.27500000000000002,"5min":0.28399999999999997,"15min":0.29399999999999998},"min":{"1min":0.19800000000000001,"5min":0.191,"15min":0.191},"max":{"1min":0.36899999999999999,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.30599999999999999},{"interface":"front","average":{"1min":0.27900000000000003,"5min":0.28499999999999998,"15min":0.29599999999999999},"min":{"1min":0.216,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":0.40799999999999997,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.32500000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051649,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:31:35 2023","interfaces":[{"interface":"back","average":{"1min":0.26100000000000001,"5min":0.25700000000000001,"15min":0.251},"min":{"1min":0.19,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.35899999999999999,"5min":0.41599999999999998,"15min":0.41599999999999998},"last":0.24199999999999999},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.27400000000000002,"15min":0.26700000000000002},"min":{"1min":0.20200000000000001,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.374,"5min":0.42899999999999999,"15min":0.42899999999999999},"last":0.22600000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:32:54.944+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:32:55.469+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:32:55.927+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)

Ignoring malparsed backtrace: 2023-03-25T04:33:13.958 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean
, 2023-03-25T04:33:13.959 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}
, 2023-03-25T04:33:13.959 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json

Ignoring malparsed backtrace: 2023-03-25T04:33:31.199 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":267,"stamp":"2023-03-25T04:33:30.166660+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002269"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:33.309748+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:29:33.309748+0000","last_peered":"2023-03-25T04:29:33.309748+0000","last_clean":"2023-03-25T04:29:33.309748+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:29:33.309748+0000","last_undegraded":"2023-03-25T04:29:33.309748+0000","last_fullsized":"2023-03-25T04:29:33.309748+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:33.309729+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:29:33.309729+0000","last_peered":"2023-03-25T04:29:33.309729+0000","last_clean":"2023-03-25T04:29:33.309729+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:29:33.309729+0000","last_undegraded":"2023-03-25T04:29:33.309729+0000","last_fullsized":"2023-03-25T04:29:33.309729+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":28,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:31:23.320894+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:31:23.320894+0000","last_peered":"2023-03-25T04:31:23.320894+0000","last_clean":"2023-03-25T04:31:23.320894+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:31:23.320894+0000","last_undegraded":"2023-03-25T04:31:23.320894+0000","last_fullsized":"2023-03-25T04:31:23.320894+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":28,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:20.554710+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:29:20.554710+0000","last_peered":"2023-03-25T04:29:20.554710+0000","last_clean":"2023-03-25T04:29:20.554710+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:29:20.554710+0000","last_undegraded":"2023-03-25T04:29:20.554710+0000","last_fullsized":"2023-03-25T04:29:20.554710+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309216+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:29:28.309216+0000","last_peered":"2023-03-25T04:29:28.309216+0000","last_clean":"2023-03-25T04:29:28.309216+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:29:28.309216+0000","last_undegraded":"2023-03-25T04:29:28.309216+0000","last_fullsized":"2023-03-25T04:29:28.309216+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309233+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:29:28.309233+0000","last_peered":"2023-03-25T04:29:28.309233+0000","last_clean":"2023-03-25T04:29:28.309233+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:29:28.309233+0000","last_undegraded":"2023-03-25T04:29:28.309233+0000","last_fullsized":"2023-03-25T04:29:28.309233+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117068,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:32:46 2023","interfaces":[{"interface":"back","average":{"1min":0.27000000000000002,"5min":0.27900000000000003,"15min":0.29199999999999998},"min":{"1min":0.17899999999999999,"5min":0.17899999999999999,"15min":0.17899999999999999},"max":{"1min":0.35599999999999998,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.28799999999999998},{"interface":"front","average":{"1min":0.26800000000000002,"5min":0.27800000000000002,"15min":0.29399999999999998},"min":{"1min":0.19900000000000001,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":0.377,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.315}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051660,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:32:36 2023","interfaces":[{"interface":"back","average":{"1min":0.248,"5min":0.25700000000000001,"15min":0.251},"min":{"1min":0.17000000000000001,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"last":0.22700000000000001},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.27600000000000002,"15min":0.26800000000000002},"min":{"1min":0.192,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"last":0.20399999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:33:31.201 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:33:31.400+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)

Ignoring malparsed backtrace: 2023-03-25T04:33:47.531+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:33:47.951+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:33:48.505+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)

Ignoring malparsed backtrace: 2023-03-25T04:34:06.104 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json
, 2023-03-25T04:34:06.118+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:34:06.497 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:34:23.399 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:34:23.665+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:34:23.712 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:34:40.195+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:34:40.523+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:34:40.676 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace: 2023-03-25T04:35:03.381+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:35:04.139+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:35:04.391+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)

Ignoring malparsed backtrace: 2023-03-25T04:35:22.738 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean
, 2023-03-25T04:35:22.739 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}
, 2023-03-25T04:35:22.739 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json

Ignoring malparsed backtrace: 2023-03-25T04:35:41.393+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:35:42.352+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:35:42.376+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:35:57.384 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":340,"stamp":"2023-03-25T04:35:56.194992+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.001930"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":36,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:31:43.322894+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:31:43.322894+0000","last_peered":"2023-03-25T04:31:43.322894+0000","last_clean":"2023-03-25T04:31:43.322894+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:31:43.322894+0000","last_undegraded":"2023-03-25T04:31:43.322894+0000","last_fullsized":"2023-03-25T04:31:43.322894+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":28,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:31:43.322876+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:31:43.322876+0000","last_peered":"2023-03-25T04:31:43.322876+0000","last_clean":"2023-03-25T04:31:43.322876+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:31:43.322876+0000","last_undegraded":"2023-03-25T04:31:43.322876+0000","last_fullsized":"2023-03-25T04:31:43.322876+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:23.331998+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:33:23.331998+0000","last_peered":"2023-03-25T04:33:23.331998+0000","last_clean":"2023-03-25T04:33:23.331998+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:33:23.331998+0000","last_undegraded":"2023-03-25T04:33:23.331998+0000","last_fullsized":"2023-03-25T04:33:23.331998+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:30.576723+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:33:30.576723+0000","last_peered":"2023-03-25T04:33:30.576723+0000","last_clean":"2023-03-25T04:33:30.576723+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:33:30.576723+0000","last_undegraded":"2023-03-25T04:33:30.576723+0000","last_fullsized":"2023-03-25T04:33:30.576723+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":37,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:38.333447+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:33:38.333447+0000","last_peered":"2023-03-25T04:33:38.333447+0000","last_clean":"2023-03-25T04:33:38.333447+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:33:38.333447+0000","last_undegraded":"2023-03-25T04:33:38.333447+0000","last_fullsized":"2023-03-25T04:33:38.333447+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:38.333467+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:33:38.333467+0000","last_peered":"2023-03-25T04:33:38.333467+0000","last_clean":"2023-03-25T04:33:38.333467+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:33:38.333467+0000","last_undegraded":"2023-03-25T04:33:38.333467+0000","last_fullsized":"2023-03-25T04:33:38.333467+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117098,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:35:49 2023","interfaces":[{"interface":"back","average":{"1min":0.255,"5min":0.26000000000000001,"15min":0.28199999999999997},"min":{"1min":0.17899999999999999,"5min":0.16,"15min":0.16},"max":{"1min":0.36399999999999999,"5min":0.42599999999999999,"15min":0.45100000000000001},"last":0.26100000000000001},{"interface":"front","average":{"1min":0.26000000000000001,"5min":0.26700000000000002,"15min":0.28599999999999998},"min":{"1min":0.188,"5min":0.17999999999999999,"15min":0.17499999999999999},"max":{"1min":0.39100000000000001,"5min":0.45200000000000001,"15min":0.46300000000000002},"last":0.28399999999999997}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051689,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:35:42 2023","interfaces":[{"interface":"back","average":{"1min":0.29399999999999998,"5min":0.27200000000000002,"15min":0.25800000000000001},"min":{"1min":0.17299999999999999,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.41099999999999998,"5min":0.42699999999999999,"15min":0.42699999999999999},"last":0.20399999999999999},{"interface":"front","average":{"1min":0.30299999999999999,"5min":0.28399999999999997,"15min":0.27300000000000002},"min":{"1min":0.19700000000000001,"5min":0.192,"15min":0.16600000000000001},"max":{"1min":0.43099999999999999,"5min":0.45800000000000002,"15min":0.45800000000000002},"last":0.22}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:35:57.624+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)

Ignoring malparsed backtrace: 2023-03-25T04:36:17.226 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:36:17.239 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":350,"stamp":"2023-03-25T04:36:16.198866+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002148"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":37,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:53.335026+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:33:53.335026+0000","last_peered":"2023-03-25T04:33:53.335026+0000","last_clean":"2023-03-25T04:33:53.335026+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:33:53.335026+0000","last_undegraded":"2023-03-25T04:33:53.335026+0000","last_fullsized":"2023-03-25T04:33:53.335026+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:53.335009+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:33:53.335009+0000","last_peered":"2023-03-25T04:33:53.335009+0000","last_clean":"2023-03-25T04:33:53.335009+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:33:53.335009+0000","last_undegraded":"2023-03-25T04:33:53.335009+0000","last_fullsized":"2023-03-25T04:33:53.335009+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:23.331998+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:33:23.331998+0000","last_peered":"2023-03-25T04:33:23.331998+0000","last_clean":"2023-03-25T04:33:23.331998+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:33:23.331998+0000","last_undegraded":"2023-03-25T04:33:23.331998+0000","last_fullsized":"2023-03-25T04:33:23.331998+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:30.576723+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:33:30.576723+0000","last_peered":"2023-03-25T04:33:30.576723+0000","last_clean":"2023-03-25T04:33:30.576723+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:33:30.576723+0000","last_undegraded":"2023-03-25T04:33:30.576723+0000","last_fullsized":"2023-03-25T04:33:30.576723+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":37,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:38.333447+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:33:38.333447+0000","last_peered":"2023-03-25T04:33:38.333447+0000","last_clean":"2023-03-25T04:33:38.333447+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:33:38.333447+0000","last_undegraded":"2023-03-25T04:33:38.333447+0000","last_fullsized":"2023-03-25T04:33:38.333447+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:38.333467+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:33:38.333467+0000","last_peered":"2023-03-25T04:33:38.333467+0000","last_clean":"2023-03-25T04:33:38.333467+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:33:38.333467+0000","last_undegraded":"2023-03-25T04:33:38.333467+0000","last_fullsized":"2023-03-25T04:33:38.333467+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117102,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:35:49 2023","interfaces":[{"interface":"back","average":{"1min":0.255,"5min":0.26000000000000001,"15min":0.28199999999999997},"min":{"1min":0.17899999999999999,"5min":0.16,"15min":0.16},"max":{"1min":0.36399999999999999,"5min":0.42599999999999999,"15min":0.45100000000000001},"last":0.28399999999999997},{"interface":"front","average":{"1min":0.26000000000000001,"5min":0.26700000000000002,"15min":0.28599999999999998},"min":{"1min":0.188,"5min":0.17999999999999999,"15min":0.17499999999999999},"max":{"1min":0.39100000000000001,"5min":0.45200000000000001,"15min":0.46300000000000002},"last":0.30299999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051693,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:35:42 2023","interfaces":[{"interface":"back","average":{"1min":0.29399999999999998,"5min":0.27200000000000002,"15min":0.25800000000000001},"min":{"1min":0.17299999999999999,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.41099999999999998,"5min":0.42699999999999999,"15min":0.42699999999999999},"last":0.35099999999999998},{"interface":"front","average":{"1min":0.30299999999999999,"5min":0.28399999999999997,"15min":0.27300000000000002},"min":{"1min":0.19700000000000001,"5min":0.192,"15min":0.16600000000000001},"max":{"1min":0.43099999999999999,"5min":0.45800000000000002,"15min":0.45800000000000002},"last":0.32900000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:36:34.556+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:36:34.717+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:36:35.583+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)

Ignoring malparsed backtrace: 2023-03-25T04:36:52.680 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json
, 2023-03-25T04:36:52.701+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:36:53.067 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:37:09.963 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:37:10.273 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all

Ignoring malparsed backtrace: 2023-03-25T04:37:26.856+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:37:27.288 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:37:27.538+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:37:41.903 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":392,"stamp":"2023-03-25T04:37:40.213373+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002012"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":37,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:53.335026+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:33:53.335026+0000","last_peered":"2023-03-25T04:33:53.335026+0000","last_clean":"2023-03-25T04:33:53.335026+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:33:53.335026+0000","last_undegraded":"2023-03-25T04:33:53.335026+0000","last_fullsized":"2023-03-25T04:33:53.335026+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:53.335009+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:33:53.335009+0000","last_peered":"2023-03-25T04:33:53.335009+0000","last_clean":"2023-03-25T04:33:53.335009+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:33:53.335009+0000","last_undegraded":"2023-03-25T04:33:53.335009+0000","last_fullsized":"2023-03-25T04:33:53.335009+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:23.343585+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:35:23.343585+0000","last_peered":"2023-03-25T04:35:23.343585+0000","last_clean":"2023-03-25T04:35:23.343585+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:35:23.343585+0000","last_undegraded":"2023-03-25T04:35:23.343585+0000","last_fullsized":"2023-03-25T04:35:23.343585+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:30.576723+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:33:30.576723+0000","last_peered":"2023-03-25T04:33:30.576723+0000","last_clean":"2023-03-25T04:33:30.576723+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:33:30.576723+0000","last_undegraded":"2023-03-25T04:33:30.576723+0000","last_fullsized":"2023-03-25T04:33:30.576723+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":37,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:38.333447+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:33:38.333447+0000","last_peered":"2023-03-25T04:33:38.333447+0000","last_clean":"2023-03-25T04:33:38.333447+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:33:38.333447+0000","last_undegraded":"2023-03-25T04:33:38.333447+0000","last_fullsized":"2023-03-25T04:33:38.333447+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:38.333467+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:33:38.333467+0000","last_peered":"2023-03-25T04:33:38.333467+0000","last_clean":"2023-03-25T04:33:38.333467+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:33:38.333467+0000","last_undegraded":"2023-03-25T04:33:38.333467+0000","last_fullsized":"2023-03-25T04:33:38.333467+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117119,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:36:50 2023","interfaces":[{"interface":"back","average":{"1min":0.254,"5min":0.25600000000000001,"15min":0.27900000000000003},"min":{"1min":0.20499999999999999,"5min":0.16,"15min":0.16},"max":{"1min":0.371,"5min":0.42599999999999999,"15min":0.45100000000000001},"last":0.23000000000000001},{"interface":"front","average":{"1min":0.26800000000000002,"5min":0.26500000000000001,"15min":0.28399999999999997},"min":{"1min":0.19700000000000001,"5min":0.17999999999999999,"15min":0.17499999999999999},"max":{"1min":0.34699999999999998,"5min":0.45200000000000001,"15min":0.46300000000000002},"last":0.20899999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051710,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:36:44 2023","interfaces":[{"interface":"back","average":{"1min":0.28399999999999997,"5min":0.27700000000000002,"15min":0.26100000000000001},"min":{"1min":0.192,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.40899999999999997,"5min":0.42699999999999999,"15min":0.42699999999999999},"last":0.39700000000000002},{"interface":"front","average":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.27500000000000002},"min":{"1min":0.182,"5min":0.182,"15min":0.16600000000000001},"max":{"1min":0.439,"5min":0.45800000000000002,"15min":0.45800000000000002},"last":0.439}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:37:42.400+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)

Ignoring malparsed backtrace: 2023-03-25T04:38:04.766+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:38:04.922 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:38:05.246 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:38:19.605 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":411,"stamp":"2023-03-25T04:38:18.220966+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002368"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":38,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:36:03.347070+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:36:03.347070+0000","last_peered":"2023-03-25T04:36:03.347070+0000","last_clean":"2023-03-25T04:36:03.347070+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:36:03.347070+0000","last_undegraded":"2023-03-25T04:36:03.347070+0000","last_fullsized":"2023-03-25T04:36:03.347070+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:36:03.347025+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:36:03.347025+0000","last_peered":"2023-03-25T04:36:03.347025+0000","last_clean":"2023-03-25T04:36:03.347025+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:36:03.347025+0000","last_undegraded":"2023-03-25T04:36:03.347025+0000","last_fullsized":"2023-03-25T04:36:03.347025+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:23.343585+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:35:23.343585+0000","last_peered":"2023-03-25T04:35:23.343585+0000","last_clean":"2023-03-25T04:35:23.343585+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:35:23.343585+0000","last_undegraded":"2023-03-25T04:35:23.343585+0000","last_fullsized":"2023-03-25T04:35:23.343585+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:35.587025+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:35:35.587025+0000","last_peered":"2023-03-25T04:35:35.587025+0000","last_clean":"2023-03-25T04:35:35.587025+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:35:35.587025+0000","last_undegraded":"2023-03-25T04:35:35.587025+0000","last_fullsized":"2023-03-25T04:35:35.587025+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":38,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:43.345308+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:35:43.345308+0000","last_peered":"2023-03-25T04:35:43.345308+0000","last_clean":"2023-03-25T04:35:43.345308+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:35:43.345308+0000","last_undegraded":"2023-03-25T04:35:43.345308+0000","last_fullsized":"2023-03-25T04:35:43.345308+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:43.345324+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:35:43.345324+0000","last_peered":"2023-03-25T04:35:43.345324+0000","last_clean":"2023-03-25T04:35:43.345324+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:35:43.345324+0000","last_undegraded":"2023-03-25T04:35:43.345324+0000","last_fullsized":"2023-03-25T04:35:43.345324+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117127,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:37:53 2023","interfaces":[{"interface":"back","average":{"1min":0.25800000000000001,"5min":0.253,"15min":0.27700000000000002},"min":{"1min":0.184,"5min":0.16,"15min":0.16},"max":{"1min":0.44600000000000001,"5min":0.44600000000000001,"15min":0.45100000000000001},"last":0.35199999999999998},{"interface":"front","average":{"1min":0.26900000000000002,"5min":0.26500000000000001,"15min":0.28199999999999997},"min":{"1min":0.192,"5min":0.17999999999999999,"15min":0.17499999999999999},"max":{"1min":0.46400000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"last":0.33200000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051717,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:37:44 2023","interfaces":[{"interface":"back","average":{"1min":0.27300000000000002,"5min":0.28199999999999997,"15min":0.26200000000000001},"min":{"1min":0.17899999999999999,"5min":0.17299999999999999,"15min":0.17000000000000001},"max":{"1min":0.41799999999999998,"5min":0.42099999999999999,"15min":0.42699999999999999},"last":0.28699999999999998},{"interface":"front","average":{"1min":0.29399999999999998,"5min":0.29099999999999998,"15min":0.27700000000000002},"min":{"1min":0.19900000000000001,"5min":0.182,"15min":0.16600000000000001},"max":{"1min":0.441,"5min":0.441,"15min":0.45800000000000002},"last":0.26300000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:38:19.718+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:38:20.315+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)

Ignoring malparsed backtrace: 2023-03-25T04:38:44.283 INFO:teuthology.orchestra.run.smithi070.stdout:
, 2023-03-25T04:38:44.283 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":781,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":16,"modified":"2023-03-25T04:38:40.225321+0000","services":{}},"progress_events":{}}
, 2023-03-25T04:38:44.296 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now

Ignoring malparsed backtrace: 2023-03-25T04:39:02.477+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:39:02.578+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:39:03.524+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:39:20.359 INFO:teuthology.orchestra.run.smithi070.stdout:
, 2023-03-25T04:39:20.359 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":818,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":16,"modified":"2023-03-25T04:38:40.225321+0000","services":{}},"progress_events":{}}
, 2023-03-25T04:39:20.372 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now

Ignoring malparsed backtrace: 2023-03-25T04:39:37.527 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:39:37.541 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":450,"stamp":"2023-03-25T04:39:36.235332+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002086"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":38,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:36:03.347070+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:36:03.347070+0000","last_peered":"2023-03-25T04:36:03.347070+0000","last_clean":"2023-03-25T04:36:03.347070+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:36:03.347070+0000","last_undegraded":"2023-03-25T04:36:03.347070+0000","last_fullsized":"2023-03-25T04:36:03.347070+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:36:03.347025+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:36:03.347025+0000","last_peered":"2023-03-25T04:36:03.347025+0000","last_clean":"2023-03-25T04:36:03.347025+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:36:03.347025+0000","last_undegraded":"2023-03-25T04:36:03.347025+0000","last_fullsized":"2023-03-25T04:36:03.347025+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:23.353778+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:37:23.353778+0000","last_peered":"2023-03-25T04:37:23.353778+0000","last_clean":"2023-03-25T04:37:23.353778+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:37:23.353778+0000","last_undegraded":"2023-03-25T04:37:23.353778+0000","last_fullsized":"2023-03-25T04:37:23.353778+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:35.587025+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:35:35.587025+0000","last_peered":"2023-03-25T04:35:35.587025+0000","last_clean":"2023-03-25T04:35:35.587025+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:35:35.587025+0000","last_undegraded":"2023-03-25T04:35:35.587025+0000","last_fullsized":"2023-03-25T04:35:35.587025+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":38,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:43.345308+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:35:43.345308+0000","last_peered":"2023-03-25T04:35:43.345308+0000","last_clean":"2023-03-25T04:35:43.345308+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:35:43.345308+0000","last_undegraded":"2023-03-25T04:35:43.345308+0000","last_fullsized":"2023-03-25T04:35:43.345308+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:43.345324+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:35:43.345324+0000","last_peered":"2023-03-25T04:35:43.345324+0000","last_clean":"2023-03-25T04:35:43.345324+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:35:43.345324+0000","last_undegraded":"2023-03-25T04:35:43.345324+0000","last_fullsized":"2023-03-25T04:35:43.345324+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117143,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:38:54 2023","interfaces":[{"interface":"back","average":{"1min":0.25800000000000001,"5min":0.25700000000000001,"15min":0.27400000000000002},"min":{"1min":0.192,"5min":0.17399999999999999,"15min":0.16},"max":{"1min":0.39400000000000002,"5min":0.44600000000000001,"15min":0.45100000000000001},"last":0.26500000000000001},{"interface":"front","average":{"1min":0.25900000000000001,"5min":0.26700000000000002,"15min":0.27900000000000003},"min":{"1min":0.19500000000000001,"5min":0.188,"15min":0.17499999999999999},"max":{"1min":0.41499999999999998,"5min":0.46400000000000002,"15min":0.46400000000000002},"last":0.29099999999999998}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051733,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:38:45 2023","interfaces":[{"interface":"back","average":{"1min":0.27300000000000002,"5min":0.28100000000000003,"15min":0.26400000000000001},"min":{"1min":0.19400000000000001,"5min":0.17299999999999999,"15min":0.17000000000000001},"max":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"last":0.27300000000000002},{"interface":"front","average":{"1min":0.28799999999999998,"5min":0.29099999999999998,"15min":0.27800000000000002},"min":{"1min":0.19800000000000001,"5min":0.182,"15min":0.16600000000000001},"max":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"last":0.308}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:39:54.416+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:39:54.455 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:39:54.780 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:40:09.288+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:40:09.389+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:40:10.267+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)

Ignoring malparsed backtrace: 2023-03-25T04:40:28.984 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":476,"stamp":"2023-03-25T04:40:28.245911+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002693"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":39,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:38:13.358968+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:38:13.358968+0000","last_peered":"2023-03-25T04:38:13.358968+0000","last_clean":"2023-03-25T04:38:13.358968+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:38:13.358968+0000","last_undegraded":"2023-03-25T04:38:13.358968+0000","last_fullsized":"2023-03-25T04:38:13.358968+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:38:13.358922+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:38:13.358922+0000","last_peered":"2023-03-25T04:38:13.358922+0000","last_clean":"2023-03-25T04:38:13.358922+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:38:13.358922+0000","last_undegraded":"2023-03-25T04:38:13.358922+0000","last_fullsized":"2023-03-25T04:38:13.358922+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:23.353778+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:37:23.353778+0000","last_peered":"2023-03-25T04:37:23.353778+0000","last_clean":"2023-03-25T04:37:23.353778+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:37:23.353778+0000","last_undegraded":"2023-03-25T04:37:23.353778+0000","last_fullsized":"2023-03-25T04:37:23.353778+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:40.598579+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:37:40.598579+0000","last_peered":"2023-03-25T04:37:40.598579+0000","last_clean":"2023-03-25T04:37:40.598579+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:37:40.598579+0000","last_undegraded":"2023-03-25T04:37:40.598579+0000","last_fullsized":"2023-03-25T04:37:40.598579+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":39,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:48.356344+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:37:48.356344+0000","last_peered":"2023-03-25T04:37:48.356344+0000","last_clean":"2023-03-25T04:37:48.356344+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:37:48.356344+0000","last_undegraded":"2023-03-25T04:37:48.356344+0000","last_fullsized":"2023-03-25T04:37:48.356344+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:48.356365+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:37:48.356365+0000","last_peered":"2023-03-25T04:37:48.356365+0000","last_clean":"2023-03-25T04:37:48.356365+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:37:48.356365+0000","last_undegraded":"2023-03-25T04:37:48.356365+0000","last_fullsized":"2023-03-25T04:37:48.356365+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117153,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:39:54 2023","interfaces":[{"interface":"back","average":{"1min":0.25900000000000001,"5min":0.25600000000000001,"15min":0.27100000000000002},"min":{"1min":0.188,"5min":0.17899999999999999,"15min":0.16},"max":{"1min":0.41999999999999998,"5min":0.44600000000000001,"15min":0.45100000000000001},"last":0.255},{"interface":"front","average":{"1min":0.27500000000000002,"5min":0.26600000000000001,"15min":0.27700000000000002},"min":{"1min":0.17699999999999999,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.45600000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"last":0.26800000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051743,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:39:47 2023","interfaces":[{"interface":"back","average":{"1min":0.26300000000000001,"5min":0.27700000000000002,"15min":0.26500000000000001},"min":{"1min":0.187,"5min":0.17299999999999999,"15min":0.17000000000000001},"max":{"1min":0.433,"5min":0.433,"15min":0.433},"last":0.36599999999999999},{"interface":"front","average":{"1min":0.27100000000000002,"5min":0.28799999999999998,"15min":0.27900000000000003},"min":{"1min":0.20100000000000001,"5min":0.182,"15min":0.16600000000000001},"max":{"1min":0.48599999999999999,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.34999999999999998}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:40:28.986 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean
, 2023-03-25T04:40:28.986 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}

Ignoring malparsed backtrace: 2023-03-25T04:40:47.229+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:40:47.503+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:40:48.251+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)

Ignoring malparsed backtrace: 2023-03-25T04:41:03.267 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:41:03.578 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all

Ignoring malparsed backtrace: 2023-03-25T04:41:21.347 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":502,"stamp":"2023-03-25T04:41:20.256204+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002342"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":39,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:38:13.358968+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:38:13.358968+0000","last_peered":"2023-03-25T04:38:13.358968+0000","last_clean":"2023-03-25T04:38:13.358968+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:38:13.358968+0000","last_undegraded":"2023-03-25T04:38:13.358968+0000","last_fullsized":"2023-03-25T04:38:13.358968+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:38:13.358922+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:38:13.358922+0000","last_peered":"2023-03-25T04:38:13.358922+0000","last_clean":"2023-03-25T04:38:13.358922+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:38:13.358922+0000","last_undegraded":"2023-03-25T04:38:13.358922+0000","last_fullsized":"2023-03-25T04:38:13.358922+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:23.353778+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:37:23.353778+0000","last_peered":"2023-03-25T04:37:23.353778+0000","last_clean":"2023-03-25T04:37:23.353778+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:37:23.353778+0000","last_undegraded":"2023-03-25T04:37:23.353778+0000","last_fullsized":"2023-03-25T04:37:23.353778+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:40.598579+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:37:40.598579+0000","last_peered":"2023-03-25T04:37:40.598579+0000","last_clean":"2023-03-25T04:37:40.598579+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:37:40.598579+0000","last_undegraded":"2023-03-25T04:37:40.598579+0000","last_fullsized":"2023-03-25T04:37:40.598579+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":39,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:48.356344+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:37:48.356344+0000","last_peered":"2023-03-25T04:37:48.356344+0000","last_clean":"2023-03-25T04:37:48.356344+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:37:48.356344+0000","last_undegraded":"2023-03-25T04:37:48.356344+0000","last_fullsized":"2023-03-25T04:37:48.356344+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:48.356365+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:37:48.356365+0000","last_peered":"2023-03-25T04:37:48.356365+0000","last_clean":"2023-03-25T04:37:48.356365+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:37:48.356365+0000","last_undegraded":"2023-03-25T04:37:48.356365+0000","last_fullsized":"2023-03-25T04:37:48.356365+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117163,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:40:56 2023","interfaces":[{"interface":"back","average":{"1min":0.28799999999999998,"5min":0.26300000000000001,"15min":0.27100000000000002},"min":{"1min":0.188,"5min":0.184,"15min":0.16},"max":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.17499999999999999},{"interface":"front","average":{"1min":0.29599999999999999,"5min":0.27300000000000002,"15min":0.27700000000000002},"min":{"1min":0.20599999999999999,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.43099999999999999,"5min":0.46400000000000002,"15min":0.46400000000000002},"last":0.20699999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051754,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:40:52 2023","interfaces":[{"interface":"back","average":{"1min":0.28599999999999998,"5min":0.27500000000000002,"15min":0.26700000000000002},"min":{"1min":0.17899999999999999,"5min":0.17899999999999999,"15min":0.17000000000000001},"max":{"1min":0.44,"5min":0.44,"15min":0.44},"last":0.23699999999999999},{"interface":"front","average":{"1min":0.29799999999999999,"5min":0.28699999999999998,"15min":0.28100000000000003},"min":{"1min":0.20300000000000001,"5min":0.182,"15min":0.16600000000000001},"max":{"1min":0.47599999999999998,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.28000000000000003}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:41:21.349 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean
, 2023-03-25T04:41:21.349 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}

Ignoring malparsed backtrace: 2023-03-25T04:41:39.867+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:41:40.383+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:41:40.905+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:41:58.219 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json
, 2023-03-25T04:41:58.319+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:41:58.644 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:42:18.193 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":530,"stamp":"2023-03-25T04:42:16.266659+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002356"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":39,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:38:13.358968+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:38:13.358968+0000","last_peered":"2023-03-25T04:38:13.358968+0000","last_clean":"2023-03-25T04:38:13.358968+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:38:13.358968+0000","last_undegraded":"2023-03-25T04:38:13.358968+0000","last_fullsized":"2023-03-25T04:38:13.358968+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:38:13.358922+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:38:13.358922+0000","last_peered":"2023-03-25T04:38:13.358922+0000","last_clean":"2023-03-25T04:38:13.358922+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:38:13.358922+0000","last_undegraded":"2023-03-25T04:38:13.358922+0000","last_fullsized":"2023-03-25T04:38:13.358922+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:23.364593+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:39:23.364593+0000","last_peered":"2023-03-25T04:39:23.364593+0000","last_clean":"2023-03-25T04:39:23.364593+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:39:23.364593+0000","last_undegraded":"2023-03-25T04:39:23.364593+0000","last_fullsized":"2023-03-25T04:39:23.364593+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:45.608475+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:39:45.608475+0000","last_peered":"2023-03-25T04:39:45.608475+0000","last_clean":"2023-03-25T04:39:45.608475+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:39:45.608475+0000","last_undegraded":"2023-03-25T04:39:45.608475+0000","last_fullsized":"2023-03-25T04:39:45.608475+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":40,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:53.367398+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:39:53.367398+0000","last_peered":"2023-03-25T04:39:53.367398+0000","last_clean":"2023-03-25T04:39:53.367398+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:39:53.367398+0000","last_undegraded":"2023-03-25T04:39:53.367398+0000","last_fullsized":"2023-03-25T04:39:53.367398+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:53.367424+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:39:53.367424+0000","last_peered":"2023-03-25T04:39:53.367424+0000","last_clean":"2023-03-25T04:39:53.367424+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:39:53.367424+0000","last_undegraded":"2023-03-25T04:39:53.367424+0000","last_fullsized":"2023-03-25T04:39:53.367424+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117175,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:41:58 2023","interfaces":[{"interface":"back","average":{"1min":0.25800000000000001,"5min":0.26400000000000001,"15min":0.26800000000000002},"min":{"1min":0.17499999999999999,"5min":0.17499999999999999,"15min":0.16},"max":{"1min":0.39100000000000001,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.22900000000000001},{"interface":"front","average":{"1min":0.26300000000000001,"5min":0.27200000000000002,"15min":0.27400000000000002},"min":{"1min":0.20599999999999999,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.35699999999999998,"5min":0.46400000000000002,"15min":0.46400000000000002},"last":0.26400000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051765,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:41:52 2023","interfaces":[{"interface":"back","average":{"1min":0.26500000000000001,"5min":0.27200000000000002,"15min":0.26800000000000002},"min":{"1min":0.17000000000000001,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.39200000000000002,"5min":0.44,"15min":0.44},"last":0.23899999999999999},{"interface":"front","average":{"1min":0.28100000000000003,"5min":0.28599999999999998,"15min":0.28199999999999997},"min":{"1min":0.19400000000000001,"5min":0.19400000000000001,"15min":0.16600000000000001},"max":{"1min":0.41399999999999998,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.254}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:42:18.297+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:42:18.958+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:42:35.512 INFO:teuthology.orchestra.run.smithi070.stdout:
, 2023-03-25T04:42:35.512 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":1013,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":17,"modified":"2023-03-25T04:41:54.262744+0000","services":{}},"progress_events":{}}
, 2023-03-25T04:42:35.525 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:42:52.741 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":548,"stamp":"2023-03-25T04:42:52.273601+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002221"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":40,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:40:23.370056+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:40:23.370056+0000","last_peered":"2023-03-25T04:40:23.370056+0000","last_clean":"2023-03-25T04:40:23.370056+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:40:23.370056+0000","last_undegraded":"2023-03-25T04:40:23.370056+0000","last_fullsized":"2023-03-25T04:40:23.370056+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:40:23.370039+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:40:23.370039+0000","last_peered":"2023-03-25T04:40:23.370039+0000","last_clean":"2023-03-25T04:40:23.370039+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:40:23.370039+0000","last_undegraded":"2023-03-25T04:40:23.370039+0000","last_fullsized":"2023-03-25T04:40:23.370039+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:23.364593+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:39:23.364593+0000","last_peered":"2023-03-25T04:39:23.364593+0000","last_clean":"2023-03-25T04:39:23.364593+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:39:23.364593+0000","last_undegraded":"2023-03-25T04:39:23.364593+0000","last_fullsized":"2023-03-25T04:39:23.364593+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:45.608475+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:39:45.608475+0000","last_peered":"2023-03-25T04:39:45.608475+0000","last_clean":"2023-03-25T04:39:45.608475+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:39:45.608475+0000","last_undegraded":"2023-03-25T04:39:45.608475+0000","last_fullsized":"2023-03-25T04:39:45.608475+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":40,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:53.367398+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:39:53.367398+0000","last_peered":"2023-03-25T04:39:53.367398+0000","last_clean":"2023-03-25T04:39:53.367398+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:39:53.367398+0000","last_undegraded":"2023-03-25T04:39:53.367398+0000","last_fullsized":"2023-03-25T04:39:53.367398+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:53.367424+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:39:53.367424+0000","last_peered":"2023-03-25T04:39:53.367424+0000","last_clean":"2023-03-25T04:39:53.367424+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:39:53.367424+0000","last_undegraded":"2023-03-25T04:39:53.367424+0000","last_fullsized":"2023-03-25T04:39:53.367424+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117182,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:41:58 2023","interfaces":[{"interface":"back","average":{"1min":0.25800000000000001,"5min":0.26400000000000001,"15min":0.26800000000000002},"min":{"1min":0.17499999999999999,"5min":0.17499999999999999,"15min":0.16},"max":{"1min":0.39100000000000001,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.19600000000000001},{"interface":"front","average":{"1min":0.26300000000000001,"5min":0.27200000000000002,"15min":0.27400000000000002},"min":{"1min":0.20599999999999999,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.35699999999999998,"5min":0.46400000000000002,"15min":0.46400000000000002},"last":0.219}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051773,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:41:52 2023","interfaces":[{"interface":"back","average":{"1min":0.26500000000000001,"5min":0.27200000000000002,"15min":0.26800000000000002},"min":{"1min":0.17000000000000001,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.39200000000000002,"5min":0.44,"15min":0.44},"last":0.19},{"interface":"front","average":{"1min":0.28100000000000003,"5min":0.28599999999999998,"15min":0.28199999999999997},"min":{"1min":0.19400000000000001,"5min":0.19400000000000001,"15min":0.16600000000000001},"max":{"1min":0.41399999999999998,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.17599999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:42:52.742 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean

Ignoring malparsed backtrace: 2023-03-25T04:43:09.973 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:43:09.986 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":556,"stamp":"2023-03-25T04:43:08.276556+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002359"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":40,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:40:23.370056+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:40:23.370056+0000","last_peered":"2023-03-25T04:40:23.370056+0000","last_clean":"2023-03-25T04:40:23.370056+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:40:23.370056+0000","last_undegraded":"2023-03-25T04:40:23.370056+0000","last_fullsized":"2023-03-25T04:40:23.370056+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:40:23.370039+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:40:23.370039+0000","last_peered":"2023-03-25T04:40:23.370039+0000","last_clean":"2023-03-25T04:40:23.370039+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:40:23.370039+0000","last_undegraded":"2023-03-25T04:40:23.370039+0000","last_fullsized":"2023-03-25T04:40:23.370039+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:23.364593+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:39:23.364593+0000","last_peered":"2023-03-25T04:39:23.364593+0000","last_clean":"2023-03-25T04:39:23.364593+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:39:23.364593+0000","last_undegraded":"2023-03-25T04:39:23.364593+0000","last_fullsized":"2023-03-25T04:39:23.364593+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:45.608475+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:39:45.608475+0000","last_peered":"2023-03-25T04:39:45.608475+0000","last_clean":"2023-03-25T04:39:45.608475+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:39:45.608475+0000","last_undegraded":"2023-03-25T04:39:45.608475+0000","last_fullsized":"2023-03-25T04:39:45.608475+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":40,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:53.367398+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:39:53.367398+0000","last_peered":"2023-03-25T04:39:53.367398+0000","last_clean":"2023-03-25T04:39:53.367398+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:39:53.367398+0000","last_undegraded":"2023-03-25T04:39:53.367398+0000","last_fullsized":"2023-03-25T04:39:53.367398+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:53.367424+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:39:53.367424+0000","last_peered":"2023-03-25T04:39:53.367424+0000","last_clean":"2023-03-25T04:39:53.367424+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:39:53.367424+0000","last_undegraded":"2023-03-25T04:39:53.367424+0000","last_fullsized":"2023-03-25T04:39:53.367424+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117185,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:43:00 2023","interfaces":[{"interface":"back","average":{"1min":0.27400000000000002,"5min":0.26700000000000002,"15min":0.26600000000000001},"min":{"1min":0.19600000000000001,"5min":0.17499999999999999,"15min":0.16},"max":{"1min":0.42999999999999999,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.27800000000000002},{"interface":"front","average":{"1min":0.28199999999999997,"5min":0.27500000000000002,"15min":0.27300000000000002},"min":{"1min":0.185,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.47199999999999998,"5min":0.47199999999999998,"15min":0.47199999999999998},"last":0.247}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051776,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:42:54 2023","interfaces":[{"interface":"back","average":{"1min":0.26300000000000001,"5min":0.27000000000000002,"15min":0.26900000000000002},"min":{"1min":0.17399999999999999,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.438,"5min":0.44,"15min":0.44},"last":0.20100000000000001},{"interface":"front","average":{"1min":0.27400000000000002,"5min":0.28199999999999997,"15min":0.28299999999999997},"min":{"1min":0.17599999999999999,"5min":0.17599999999999999,"15min":0.16600000000000001},"max":{"1min":0.41599999999999998,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.23499999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:43:25.042+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:43:25.411+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:43:26.013+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:43:47.937+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:43:48.534+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:43:48.889+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:44:09.318 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:44:09.331 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":586,"stamp":"2023-03-25T04:44:08.288933+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002697"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":40,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:40:23.370056+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:40:23.370056+0000","last_peered":"2023-03-25T04:40:23.370056+0000","last_clean":"2023-03-25T04:40:23.370056+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:40:23.370056+0000","last_undegraded":"2023-03-25T04:40:23.370056+0000","last_fullsized":"2023-03-25T04:40:23.370056+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:40:23.370039+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:40:23.370039+0000","last_peered":"2023-03-25T04:40:23.370039+0000","last_clean":"2023-03-25T04:40:23.370039+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:40:23.370039+0000","last_undegraded":"2023-03-25T04:40:23.370039+0000","last_fullsized":"2023-03-25T04:40:23.370039+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:23.375106+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:41:23.375106+0000","last_peered":"2023-03-25T04:41:23.375106+0000","last_clean":"2023-03-25T04:41:23.375106+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:41:23.375106+0000","last_undegraded":"2023-03-25T04:41:23.375106+0000","last_fullsized":"2023-03-25T04:41:23.375106+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:50.618930+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:41:50.618930+0000","last_peered":"2023-03-25T04:41:50.618930+0000","last_clean":"2023-03-25T04:41:50.618930+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:41:50.618930+0000","last_undegraded":"2023-03-25T04:41:50.618930+0000","last_fullsized":"2023-03-25T04:41:50.618930+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":41,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:58.377821+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:41:58.377821+0000","last_peered":"2023-03-25T04:41:58.377821+0000","last_clean":"2023-03-25T04:41:58.377821+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:41:58.377821+0000","last_undegraded":"2023-03-25T04:41:58.377821+0000","last_fullsized":"2023-03-25T04:41:58.377821+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:58.377836+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:41:58.377836+0000","last_peered":"2023-03-25T04:41:58.377836+0000","last_clean":"2023-03-25T04:41:58.377836+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:41:58.377836+0000","last_undegraded":"2023-03-25T04:41:58.377836+0000","last_fullsized":"2023-03-25T04:41:58.377836+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117197,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:44:01 2023","interfaces":[{"interface":"back","average":{"1min":0.27200000000000002,"5min":0.27000000000000002,"15min":0.26400000000000001},"min":{"1min":0.20399999999999999,"5min":0.17499999999999999,"15min":0.16},"max":{"1min":0.38400000000000001,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.245},{"interface":"front","average":{"1min":0.27600000000000002,"5min":0.27800000000000002,"15min":0.27100000000000002},"min":{"1min":0.23100000000000001,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.40400000000000003,"5min":0.47199999999999998,"15min":0.47199999999999998},"last":0.27500000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051788,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:43:55 2023","interfaces":[{"interface":"back","average":{"1min":0.26100000000000001,"5min":0.26700000000000002,"15min":0.27000000000000002},"min":{"1min":0.17799999999999999,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.374,"5min":0.44,"15min":0.44},"last":0.30299999999999999},{"interface":"front","average":{"1min":0.28299999999999997,"5min":0.28100000000000003,"15min":0.28399999999999997},"min":{"1min":0.20899999999999999,"5min":0.17599999999999999,"15min":0.16600000000000001},"max":{"1min":0.40100000000000002,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.27800000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:44:28.923 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":1126,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":19,"modified":"2023-03-25T04:43:20.278959+0000","services":{}},"progress_events":{}}
, 2023-03-25T04:44:28.936 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now
, 2023-03-25T04:44:28.937 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace: 2023-03-25T04:44:48.786 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:44:48.800 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":606,"stamp":"2023-03-25T04:44:48.296805+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002182"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":41,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:42:34.382196+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:42:34.382196+0000","last_peered":"2023-03-25T04:42:34.382196+0000","last_clean":"2023-03-25T04:42:34.382196+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:42:34.382196+0000","last_undegraded":"2023-03-25T04:42:34.382196+0000","last_fullsized":"2023-03-25T04:42:34.382196+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:42:29.381744+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:42:29.381744+0000","last_peered":"2023-03-25T04:42:29.381744+0000","last_clean":"2023-03-25T04:42:29.381744+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:42:29.381744+0000","last_undegraded":"2023-03-25T04:42:29.381744+0000","last_fullsized":"2023-03-25T04:42:29.381744+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:23.375106+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:41:23.375106+0000","last_peered":"2023-03-25T04:41:23.375106+0000","last_clean":"2023-03-25T04:41:23.375106+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:41:23.375106+0000","last_undegraded":"2023-03-25T04:41:23.375106+0000","last_fullsized":"2023-03-25T04:41:23.375106+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:50.618930+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:41:50.618930+0000","last_peered":"2023-03-25T04:41:50.618930+0000","last_clean":"2023-03-25T04:41:50.618930+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:41:50.618930+0000","last_undegraded":"2023-03-25T04:41:50.618930+0000","last_fullsized":"2023-03-25T04:41:50.618930+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":41,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:58.377821+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:41:58.377821+0000","last_peered":"2023-03-25T04:41:58.377821+0000","last_clean":"2023-03-25T04:41:58.377821+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:41:58.377821+0000","last_undegraded":"2023-03-25T04:41:58.377821+0000","last_fullsized":"2023-03-25T04:41:58.377821+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:58.377836+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:41:58.377836+0000","last_peered":"2023-03-25T04:41:58.377836+0000","last_clean":"2023-03-25T04:41:58.377836+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:41:58.377836+0000","last_undegraded":"2023-03-25T04:41:58.377836+0000","last_fullsized":"2023-03-25T04:41:58.377836+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117206,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:44:01 2023","interfaces":[{"interface":"back","average":{"1min":0.27200000000000002,"5min":0.27000000000000002,"15min":0.26400000000000001},"min":{"1min":0.20399999999999999,"5min":0.17499999999999999,"15min":0.16},"max":{"1min":0.38400000000000001,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.216},{"interface":"front","average":{"1min":0.27600000000000002,"5min":0.27800000000000002,"15min":0.27100000000000002},"min":{"1min":0.23100000000000001,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.40400000000000003,"5min":0.47199999999999998,"15min":0.47199999999999998},"last":0.23999999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051796,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:43:55 2023","interfaces":[{"interface":"back","average":{"1min":0.26100000000000001,"5min":0.26700000000000002,"15min":0.27000000000000002},"min":{"1min":0.17799999999999999,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.374,"5min":0.44,"15min":0.44},"last":0.25600000000000001},{"interface":"front","average":{"1min":0.28299999999999997,"5min":0.28100000000000003,"15min":0.28399999999999997},"min":{"1min":0.20899999999999999,"5min":0.17599999999999999,"15min":0.16600000000000001},"max":{"1min":0.40100000000000002,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.28000000000000003}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:45:05.762 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:45:06.073 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all

Ignoring malparsed backtrace: 2023-03-25T04:45:25.658 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":624,"stamp":"2023-03-25T04:45:24.304294+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002366"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":41,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:42:34.382196+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:42:34.382196+0000","last_peered":"2023-03-25T04:42:34.382196+0000","last_clean":"2023-03-25T04:42:34.382196+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:42:34.382196+0000","last_undegraded":"2023-03-25T04:42:34.382196+0000","last_fullsized":"2023-03-25T04:42:34.382196+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:42:29.381744+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:42:29.381744+0000","last_peered":"2023-03-25T04:42:29.381744+0000","last_clean":"2023-03-25T04:42:29.381744+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:42:29.381744+0000","last_undegraded":"2023-03-25T04:42:29.381744+0000","last_fullsized":"2023-03-25T04:42:29.381744+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:23.375106+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:41:23.375106+0000","last_peered":"2023-03-25T04:41:23.375106+0000","last_clean":"2023-03-25T04:41:23.375106+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:41:23.375106+0000","last_undegraded":"2023-03-25T04:41:23.375106+0000","last_fullsized":"2023-03-25T04:41:23.375106+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:50.618930+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:41:50.618930+0000","last_peered":"2023-03-25T04:41:50.618930+0000","last_clean":"2023-03-25T04:41:50.618930+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:41:50.618930+0000","last_undegraded":"2023-03-25T04:41:50.618930+0000","last_fullsized":"2023-03-25T04:41:50.618930+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":41,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:58.377821+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:41:58.377821+0000","last_peered":"2023-03-25T04:41:58.377821+0000","last_clean":"2023-03-25T04:41:58.377821+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:41:58.377821+0000","last_undegraded":"2023-03-25T04:41:58.377821+0000","last_fullsized":"2023-03-25T04:41:58.377821+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:58.377836+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:41:58.377836+0000","last_peered":"2023-03-25T04:41:58.377836+0000","last_clean":"2023-03-25T04:41:58.377836+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:41:58.377836+0000","last_undegraded":"2023-03-25T04:41:58.377836+0000","last_fullsized":"2023-03-25T04:41:58.377836+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117213,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:45:01 2023","interfaces":[{"interface":"back","average":{"1min":0.251,"5min":0.26800000000000002,"15min":0.26400000000000001},"min":{"1min":0.183,"5min":0.17499999999999999,"15min":0.16},"max":{"1min":0.41499999999999998,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.26900000000000002},{"interface":"front","average":{"1min":0.28000000000000003,"5min":0.27900000000000003,"15min":0.27100000000000002},"min":{"1min":0.20999999999999999,"5min":0.185,"15min":0.17499999999999999},"max":{"1min":0.44400000000000001,"5min":0.47199999999999998,"15min":0.47199999999999998},"last":0.27900000000000003}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051803,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:44:55 2023","interfaces":[{"interface":"back","average":{"1min":0.252,"5min":0.26500000000000001,"15min":0.26900000000000002},"min":{"1min":0.183,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.39700000000000002,"5min":0.44,"15min":0.44},"last":0.20699999999999999},{"interface":"front","average":{"1min":0.27000000000000002,"5min":0.28100000000000003,"15min":0.28299999999999997},"min":{"1min":0.20300000000000001,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":0.42099999999999999,"5min":0.47599999999999998,"15min":0.48599999999999999},"last":0.23599999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:45:25.659 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:45:25.969 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:45:42.735+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:45:42.744+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:45:42.897 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace: 2023-03-25T04:46:00.183 INFO:teuthology.orchestra.run.smithi070.stdout:
, 2023-03-25T04:46:00.183 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":1217,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":20,"modified":"2023-03-25T04:45:42.307700+0000","services":{"mgr":{"daemons":{"summary":"","x":{"start_epoch":0,"start_stamp":"0.000000","gid":0,"addr":"(unrecognized address family 0)/0","metadata":{},"task_status":{}}}}}},"progress_events":{}}
, 2023-03-25T04:46:00.197 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now

Ignoring malparsed backtrace: 2023-03-25T04:46:19.910+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:46:20.039 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all

Ignoring malparsed backtrace: 2023-03-25T04:46:39.611 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}
, 2023-03-25T04:46:39.611 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json
, 2023-03-25T04:46:39.613+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:46:58.485+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:46:58.764+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:46:59.519+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:47:14.132 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:47:14.146 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":678,"stamp":"2023-03-25T04:47:12.325044+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002571"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":42,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:44.393426+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:44:44.393426+0000","last_peered":"2023-03-25T04:44:44.393426+0000","last_clean":"2023-03-25T04:44:44.393426+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:44:44.393426+0000","last_undegraded":"2023-03-25T04:44:44.393426+0000","last_fullsized":"2023-03-25T04:44:44.393426+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:39.393044+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:44:39.393044+0000","last_peered":"2023-03-25T04:44:39.393044+0000","last_clean":"2023-03-25T04:44:39.393044+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:44:39.393044+0000","last_undegraded":"2023-03-25T04:44:39.393044+0000","last_fullsized":"2023-03-25T04:44:39.393044+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:43:24.386177+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:43:24.386177+0000","last_peered":"2023-03-25T04:43:24.386177+0000","last_clean":"2023-03-25T04:43:24.386177+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:43:24.386177+0000","last_undegraded":"2023-03-25T04:43:24.386177+0000","last_fullsized":"2023-03-25T04:43:24.386177+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:43:55.629798+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:43:55.629798+0000","last_peered":"2023-03-25T04:43:55.629798+0000","last_clean":"2023-03-25T04:43:55.629798+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:43:55.629798+0000","last_undegraded":"2023-03-25T04:43:55.629798+0000","last_fullsized":"2023-03-25T04:43:55.629798+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":42,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:04.389904+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:44:04.389904+0000","last_peered":"2023-03-25T04:44:04.389904+0000","last_clean":"2023-03-25T04:44:04.389904+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:44:04.389904+0000","last_undegraded":"2023-03-25T04:44:04.389904+0000","last_fullsized":"2023-03-25T04:44:04.389904+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:04.389923+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:44:04.389923+0000","last_peered":"2023-03-25T04:44:04.389923+0000","last_clean":"2023-03-25T04:44:04.389923+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:44:04.389923+0000","last_undegraded":"2023-03-25T04:44:04.389923+0000","last_fullsized":"2023-03-25T04:44:04.389923+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117235,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:47:05 2023","interfaces":[{"interface":"back","average":{"1min":0.27800000000000002,"5min":0.27000000000000002,"15min":0.26300000000000001},"min":{"1min":0.20100000000000001,"5min":0.16900000000000001,"15min":0.16},"max":{"1min":0.39200000000000002,"5min":0.42999999999999999,"15min":0.45600000000000002},"last":0.379},{"interface":"front","average":{"1min":0.28399999999999997,"5min":0.28100000000000003,"15min":0.27200000000000002},"min":{"1min":0.17799999999999999,"5min":0.17799999999999999,"15min":0.17699999999999999},"max":{"1min":0.42599999999999999,"5min":0.47199999999999998,"15min":0.47199999999999998},"last":0.39800000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051825,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:47:01 2023","interfaces":[{"interface":"back","average":{"1min":0.27300000000000002,"5min":0.26100000000000001,"15min":0.27000000000000002},"min":{"1min":0.186,"5min":0.17399999999999999,"15min":0.17000000000000001},"max":{"1min":0.40400000000000003,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.23899999999999999},{"interface":"front","average":{"1min":0.28699999999999998,"5min":0.27500000000000002,"15min":0.28299999999999997},"min":{"1min":0.186,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.48599999999999999},"last":0.19600000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:47:31.198 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json
, 2023-03-25T04:47:31.411+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:47:31.789 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:47:49.509 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":696,"stamp":"2023-03-25T04:47:48.332093+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002697"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":42,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:44.393426+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:44:44.393426+0000","last_peered":"2023-03-25T04:44:44.393426+0000","last_clean":"2023-03-25T04:44:44.393426+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:44:44.393426+0000","last_undegraded":"2023-03-25T04:44:44.393426+0000","last_fullsized":"2023-03-25T04:44:44.393426+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:39.393044+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:44:39.393044+0000","last_peered":"2023-03-25T04:44:39.393044+0000","last_clean":"2023-03-25T04:44:39.393044+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:44:39.393044+0000","last_undegraded":"2023-03-25T04:44:39.393044+0000","last_fullsized":"2023-03-25T04:44:39.393044+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:45:24.396717+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:45:24.396717+0000","last_peered":"2023-03-25T04:45:24.396717+0000","last_clean":"2023-03-25T04:45:24.396717+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:45:24.396717+0000","last_undegraded":"2023-03-25T04:45:24.396717+0000","last_fullsized":"2023-03-25T04:45:24.396717+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:43:55.629798+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:43:55.629798+0000","last_peered":"2023-03-25T04:43:55.629798+0000","last_clean":"2023-03-25T04:43:55.629798+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:43:55.629798+0000","last_undegraded":"2023-03-25T04:43:55.629798+0000","last_fullsized":"2023-03-25T04:43:55.629798+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":42,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:04.389904+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:44:04.389904+0000","last_peered":"2023-03-25T04:44:04.389904+0000","last_clean":"2023-03-25T04:44:04.389904+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:44:04.389904+0000","last_undegraded":"2023-03-25T04:44:04.389904+0000","last_fullsized":"2023-03-25T04:44:04.389904+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:04.389923+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:44:04.389923+0000","last_peered":"2023-03-25T04:44:04.389923+0000","last_clean":"2023-03-25T04:44:04.389923+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:44:04.389923+0000","last_undegraded":"2023-03-25T04:44:04.389923+0000","last_fullsized":"2023-03-25T04:44:04.389923+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117242,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:47:05 2023","interfaces":[{"interface":"back","average":{"1min":0.27800000000000002,"5min":0.27000000000000002,"15min":0.26300000000000001},"min":{"1min":0.20100000000000001,"5min":0.16900000000000001,"15min":0.16},"max":{"1min":0.39200000000000002,"5min":0.42999999999999999,"15min":0.45600000000000002},"last":0.40500000000000003},{"interface":"front","average":{"1min":0.28399999999999997,"5min":0.28100000000000003,"15min":0.27200000000000002},"min":{"1min":0.17799999999999999,"5min":0.17799999999999999,"15min":0.17699999999999999},"max":{"1min":0.42599999999999999,"5min":0.47199999999999998,"15min":0.47199999999999998},"last":0.42399999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051834,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:47:01 2023","interfaces":[{"interface":"back","average":{"1min":0.27300000000000002,"5min":0.26100000000000001,"15min":0.27000000000000002},"min":{"1min":0.186,"5min":0.17399999999999999,"15min":0.17000000000000001},"max":{"1min":0.40400000000000003,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.25600000000000001},{"interface":"front","average":{"1min":0.28699999999999998,"5min":0.27500000000000002,"15min":0.28299999999999997},"min":{"1min":0.186,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.48599999999999999},"last":0.22800000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:47:49.510 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace:     ctx.managers[config['cluster']].wait_for_clean()
,   File "/home/teuthworker/src/github.com_ceph_ceph-c_511015c23eef0684a64171b710e432448df2b7c7/qa/tasks/ceph_manager.py", line 2747, in wait_for_clean
,     assert time.time() - start < timeout, \

Ignoring malparsed backtrace:  ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
,  1: /lib64/libpthread.so.0(+0x12cf0) [0x7f1453c2fcf0]
,  2: __pthread_mutex_lock()

Ignoring malparsed backtrace: 2023-03-25T04:28:05.172 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":142,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":5,"modified":"2023-03-25T04:27:12.092098+0000","services":{}},"progress_events":{}}
, 2023-03-25T04:28:05.185 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now
, 2023-03-25T04:28:05.186 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace: 2023-03-25T04:28:28.360 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":116,"stamp":"2023-03-25T04:28:28.108007+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002578"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.314661+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:27:45.314661+0000","last_peered":"2023-03-25T04:27:45.314661+0000","last_clean":"2023-03-25T04:27:45.314661+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:27:45.314661+0000","last_undegraded":"2023-03-25T04:27:45.314661+0000","last_fullsized":"2023-03-25T04:27:45.314661+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":25,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.310745+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:27:44.310745+0000","last_peered":"2023-03-25T04:27:44.310745+0000","last_clean":"2023-03-25T04:27:44.310745+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:27:44.310745+0000","last_undegraded":"2023-03-25T04:27:44.310745+0000","last_fullsized":"2023-03-25T04:27:44.310745+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":25,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.310650+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:27:44.310650+0000","last_peered":"2023-03-25T04:27:44.310650+0000","last_clean":"2023-03-25T04:27:44.310650+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:27:44.310650+0000","last_undegraded":"2023-03-25T04:27:44.310650+0000","last_fullsized":"2023-03-25T04:27:44.310650+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":26,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.312962+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:27:44.312962+0000","last_peered":"2023-03-25T04:27:44.312962+0000","last_clean":"2023-03-25T04:27:44.312962+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:27:44.312962+0000","last_undegraded":"2023-03-25T04:27:44.312962+0000","last_fullsized":"2023-03-25T04:27:44.312962+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315064+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:27:45.315064+0000","last_peered":"2023-03-25T04:27:45.315064+0000","last_clean":"2023-03-25T04:27:45.315064+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:27:45.315064+0000","last_undegraded":"2023-03-25T04:27:45.315064+0000","last_fullsized":"2023-03-25T04:27:45.315064+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":25,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.310707+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:27:44.310707+0000","last_peered":"2023-03-25T04:27:44.310707+0000","last_clean":"2023-03-25T04:27:44.310707+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:27:44.310707+0000","last_undegraded":"2023-03-25T04:27:44.310707+0000","last_fullsized":"2023-03-25T04:27:44.310707+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117007,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.375}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051597,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.186}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:28:28.362 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean
, 2023-03-25T04:28:28.362 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:28:45.655 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":124,"stamp":"2023-03-25T04:28:44.111265+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002492"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.314661+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:27:45.314661+0000","last_peered":"2023-03-25T04:27:45.314661+0000","last_clean":"2023-03-25T04:27:45.314661+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:27:45.314661+0000","last_undegraded":"2023-03-25T04:27:45.314661+0000","last_fullsized":"2023-03-25T04:27:45.314661+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":25,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.310745+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:27:44.310745+0000","last_peered":"2023-03-25T04:27:44.310745+0000","last_clean":"2023-03-25T04:27:44.310745+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:27:44.310745+0000","last_undegraded":"2023-03-25T04:27:44.310745+0000","last_fullsized":"2023-03-25T04:27:44.310745+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":25,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.310650+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:27:44.310650+0000","last_peered":"2023-03-25T04:27:44.310650+0000","last_clean":"2023-03-25T04:27:44.310650+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:27:44.310650+0000","last_undegraded":"2023-03-25T04:27:44.310650+0000","last_fullsized":"2023-03-25T04:27:44.310650+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":26,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.312962+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:27:44.312962+0000","last_peered":"2023-03-25T04:27:44.312962+0000","last_clean":"2023-03-25T04:27:44.312962+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:27:44.312962+0000","last_undegraded":"2023-03-25T04:27:44.312962+0000","last_fullsized":"2023-03-25T04:27:44.312962+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315064+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:27:45.315064+0000","last_peered":"2023-03-25T04:27:45.315064+0000","last_clean":"2023-03-25T04:27:45.315064+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:27:45.315064+0000","last_undegraded":"2023-03-25T04:27:45.315064+0000","last_fullsized":"2023-03-25T04:27:45.315064+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":25,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.310707+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:27:44.310707+0000","last_peered":"2023-03-25T04:27:44.310707+0000","last_clean":"2023-03-25T04:27:44.310707+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:27:44.310707+0000","last_undegraded":"2023-03-25T04:27:44.310707+0000","last_fullsized":"2023-03-25T04:27:44.310707+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117011,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:28:31 2023","interfaces":[{"interface":"back","average":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.36899999999999999},{"interface":"front","average":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"min":{"1min":0.19,"5min":0.19,"15min":0.19},"max":{"1min":0.46300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.39800000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051601,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:28:24 2023","interfaces":[{"interface":"back","average":{"1min":0.249,"5min":0.249,"15min":0.249},"min":{"1min":0.17699999999999999,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"last":0.254},{"interface":"front","average":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"min":{"1min":0.20200000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"last":0.312}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:28:45.656 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace: 2023-03-25T04:29:05.171 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}
, 2023-03-25T04:29:05.171 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json
, 2023-03-25T04:29:05.561 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:29:22.465 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:29:22.773 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all

Ignoring malparsed backtrace: 2023-03-25T04:29:42.405 INFO:teuthology.orchestra.run.smithi070.stdout:
, 2023-03-25T04:29:42.406 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":240,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":9,"modified":"2023-03-25T04:29:32.120358+0000","services":{"mon":{"daemons":{"summary":"","a":{"start_epoch":0,"start_stamp":"0.000000","gid":0,"addr":"(unrecognized address family 0)/0","metadata":{},"task_status":{}}}}}},"progress_events":{}}
, 2023-03-25T04:29:42.419 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:29:59.627 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":161,"stamp":"2023-03-25T04:29:58.125186+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002398"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315468+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:27:45.315468+0000","last_peered":"2023-03-25T04:27:45.315468+0000","last_clean":"2023-03-25T04:27:45.315468+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:27:45.315468+0000","last_undegraded":"2023-03-25T04:27:45.315468+0000","last_fullsized":"2023-03-25T04:27:45.315468+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312845+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:27:45.312845+0000","last_peered":"2023-03-25T04:27:45.312845+0000","last_clean":"2023-03-25T04:27:45.312845+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:27:45.312845+0000","last_undegraded":"2023-03-25T04:27:45.312845+0000","last_fullsized":"2023-03-25T04:27:45.312845+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312882+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:27:45.312882+0000","last_peered":"2023-03-25T04:27:45.312882+0000","last_clean":"2023-03-25T04:27:45.312882+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:27:45.312882+0000","last_undegraded":"2023-03-25T04:27:45.312882+0000","last_fullsized":"2023-03-25T04:27:45.312882+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.311730+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:27:45.311730+0000","last_peered":"2023-03-25T04:27:45.311730+0000","last_clean":"2023-03-25T04:27:45.311730+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:27:45.311730+0000","last_undegraded":"2023-03-25T04:27:45.311730+0000","last_fullsized":"2023-03-25T04:27:45.311730+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315735+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:27:45.315735+0000","last_peered":"2023-03-25T04:27:45.315735+0000","last_clean":"2023-03-25T04:27:45.315735+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:27:45.315735+0000","last_undegraded":"2023-03-25T04:27:45.315735+0000","last_fullsized":"2023-03-25T04:27:45.315735+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312833+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:27:45.312833+0000","last_peered":"2023-03-25T04:27:45.312833+0000","last_clean":"2023-03-25T04:27:45.312833+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:27:45.312833+0000","last_undegraded":"2023-03-25T04:27:45.312833+0000","last_fullsized":"2023-03-25T04:27:45.312833+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117026,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:29:33 2023","interfaces":[{"interface":"back","average":{"1min":0.26300000000000001,"5min":0.29099999999999998,"15min":0.29599999999999999},"min":{"1min":0.191,"5min":0.191,"15min":0.191},"max":{"1min":0.371,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.20499999999999999},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.29599999999999999,"15min":0.29999999999999999},"min":{"1min":0.21099999999999999,"5min":0.19,"15min":0.19},"max":{"1min":0.39800000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.19500000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051616,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:29:29 2023","interfaces":[{"interface":"back","average":{"1min":0.27200000000000002,"5min":0.253,"15min":0.25},"min":{"1min":0.183,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"last":0.26400000000000001},{"interface":"front","average":{"1min":0.28499999999999998,"5min":0.26800000000000002,"15min":0.26500000000000001},"min":{"1min":0.16600000000000001,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"last":0.29899999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:29:59.629 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean

Ignoring malparsed backtrace: 2023-03-25T04:30:16.889 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:30:16.903 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":170,"stamp":"2023-03-25T04:30:16.128427+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002222"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315468+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:27:45.315468+0000","last_peered":"2023-03-25T04:27:45.315468+0000","last_clean":"2023-03-25T04:27:45.315468+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:27:45.315468+0000","last_undegraded":"2023-03-25T04:27:45.315468+0000","last_fullsized":"2023-03-25T04:27:45.315468+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312845+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:27:45.312845+0000","last_peered":"2023-03-25T04:27:45.312845+0000","last_clean":"2023-03-25T04:27:45.312845+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:27:45.312845+0000","last_undegraded":"2023-03-25T04:27:45.312845+0000","last_fullsized":"2023-03-25T04:27:45.312845+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312882+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:27:45.312882+0000","last_peered":"2023-03-25T04:27:45.312882+0000","last_clean":"2023-03-25T04:27:45.312882+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:27:45.312882+0000","last_undegraded":"2023-03-25T04:27:45.312882+0000","last_fullsized":"2023-03-25T04:27:45.312882+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.311730+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:27:45.311730+0000","last_peered":"2023-03-25T04:27:45.311730+0000","last_clean":"2023-03-25T04:27:45.311730+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:27:45.311730+0000","last_undegraded":"2023-03-25T04:27:45.311730+0000","last_fullsized":"2023-03-25T04:27:45.311730+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315735+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:27:45.315735+0000","last_peered":"2023-03-25T04:27:45.315735+0000","last_clean":"2023-03-25T04:27:45.315735+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:27:45.315735+0000","last_undegraded":"2023-03-25T04:27:45.315735+0000","last_fullsized":"2023-03-25T04:27:45.315735+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312833+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:27:45.312833+0000","last_peered":"2023-03-25T04:27:45.312833+0000","last_clean":"2023-03-25T04:27:45.312833+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:27:45.312833+0000","last_undegraded":"2023-03-25T04:27:45.312833+0000","last_fullsized":"2023-03-25T04:27:45.312833+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117030,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:29:33 2023","interfaces":[{"interface":"back","average":{"1min":0.26300000000000001,"5min":0.29099999999999998,"15min":0.29599999999999999},"min":{"1min":0.191,"5min":0.191,"15min":0.191},"max":{"1min":0.371,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.245},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.29599999999999999,"15min":0.29999999999999999},"min":{"1min":0.21099999999999999,"5min":0.19,"15min":0.19},"max":{"1min":0.39800000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.20899999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051620,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:29:29 2023","interfaces":[{"interface":"back","average":{"1min":0.27200000000000002,"5min":0.253,"15min":0.25},"min":{"1min":0.183,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"last":0.19400000000000001},{"interface":"front","average":{"1min":0.28499999999999998,"5min":0.26800000000000002,"15min":0.26500000000000001},"min":{"1min":0.16600000000000001,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"last":0.23599999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:30:31.984+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:30:32.201+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:30:32.981+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)

Ignoring malparsed backtrace: 2023-03-25T04:30:48.449 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now
, 2023-03-25T04:30:48.450 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:30:48.760 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:31:08.342 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":196,"stamp":"2023-03-25T04:31:08.139018+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002431"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315468+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:27:45.315468+0000","last_peered":"2023-03-25T04:27:45.315468+0000","last_clean":"2023-03-25T04:27:45.315468+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:27:45.315468+0000","last_undegraded":"2023-03-25T04:27:45.315468+0000","last_fullsized":"2023-03-25T04:27:45.315468+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312845+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:27:45.312845+0000","last_peered":"2023-03-25T04:27:45.312845+0000","last_clean":"2023-03-25T04:27:45.312845+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:27:45.312845+0000","last_undegraded":"2023-03-25T04:27:45.312845+0000","last_fullsized":"2023-03-25T04:27:45.312845+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312882+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:27:45.312882+0000","last_peered":"2023-03-25T04:27:45.312882+0000","last_clean":"2023-03-25T04:27:45.312882+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:27:45.312882+0000","last_undegraded":"2023-03-25T04:27:45.312882+0000","last_fullsized":"2023-03-25T04:27:45.312882+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.311730+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:27:45.311730+0000","last_peered":"2023-03-25T04:27:45.311730+0000","last_clean":"2023-03-25T04:27:45.311730+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:27:45.311730+0000","last_undegraded":"2023-03-25T04:27:45.311730+0000","last_fullsized":"2023-03-25T04:27:45.311730+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315735+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:27:45.315735+0000","last_peered":"2023-03-25T04:27:45.315735+0000","last_clean":"2023-03-25T04:27:45.315735+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:27:45.315735+0000","last_undegraded":"2023-03-25T04:27:45.315735+0000","last_fullsized":"2023-03-25T04:27:45.315735+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312833+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:27:45.312833+0000","last_peered":"2023-03-25T04:27:45.312833+0000","last_clean":"2023-03-25T04:27:45.312833+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:27:45.312833+0000","last_undegraded":"2023-03-25T04:27:45.312833+0000","last_fullsized":"2023-03-25T04:27:45.312833+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117040,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:30:37 2023","interfaces":[{"interface":"back","average":{"1min":0.28799999999999998,"5min":0.28899999999999998,"15min":0.29499999999999998},"min":{"1min":0.20200000000000001,"5min":0.191,"15min":0.191},"max":{"1min":0.433,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.29799999999999999},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.28999999999999998,"15min":0.29799999999999999},"min":{"1min":0.17499999999999999,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":0.40300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.25700000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051630,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:30:32 2023","interfaces":[{"interface":"back","average":{"1min":0.25600000000000001,"5min":0.255,"15min":0.251},"min":{"1min":0.19400000000000001,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.35699999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"last":0.22800000000000001},{"interface":"front","average":{"1min":0.28699999999999998,"5min":0.27200000000000002,"15min":0.26600000000000001},"min":{"1min":0.20599999999999999,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.39400000000000002,"5min":0.42899999999999999,"15min":0.42899999999999999},"last":0.24099999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:31:08.343 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace: 2023-03-25T04:31:24.283+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:31:25.192+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:31:25.295+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)

Ignoring malparsed backtrace: 2023-03-25T04:31:40.257 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:31:40.271 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":212,"stamp":"2023-03-25T04:31:40.144732+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002208"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315468+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:27:45.315468+0000","last_peered":"2023-03-25T04:27:45.315468+0000","last_clean":"2023-03-25T04:27:45.315468+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:27:45.315468+0000","last_undegraded":"2023-03-25T04:27:45.315468+0000","last_fullsized":"2023-03-25T04:27:45.315468+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312845+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:27:45.312845+0000","last_peered":"2023-03-25T04:27:45.312845+0000","last_clean":"2023-03-25T04:27:45.312845+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:27:45.312845+0000","last_undegraded":"2023-03-25T04:27:45.312845+0000","last_fullsized":"2023-03-25T04:27:45.312845+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:23.308607+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:29:23.308607+0000","last_peered":"2023-03-25T04:29:23.308607+0000","last_clean":"2023-03-25T04:29:23.308607+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:29:23.308607+0000","last_undegraded":"2023-03-25T04:29:23.308607+0000","last_fullsized":"2023-03-25T04:29:23.308607+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":28,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:20.554710+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:29:20.554710+0000","last_peered":"2023-03-25T04:29:20.554710+0000","last_clean":"2023-03-25T04:29:20.554710+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:29:20.554710+0000","last_undegraded":"2023-03-25T04:29:20.554710+0000","last_fullsized":"2023-03-25T04:29:20.554710+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309216+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:29:28.309216+0000","last_peered":"2023-03-25T04:29:28.309216+0000","last_clean":"2023-03-25T04:29:28.309216+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:29:28.309216+0000","last_undegraded":"2023-03-25T04:29:28.309216+0000","last_fullsized":"2023-03-25T04:29:28.309216+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309233+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:29:28.309233+0000","last_peered":"2023-03-25T04:29:28.309233+0000","last_clean":"2023-03-25T04:29:28.309233+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:29:28.309233+0000","last_undegraded":"2023-03-25T04:29:28.309233+0000","last_fullsized":"2023-03-25T04:29:28.309233+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117046,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:30:37 2023","interfaces":[{"interface":"back","average":{"1min":0.28799999999999998,"5min":0.28899999999999998,"15min":0.29499999999999998},"min":{"1min":0.20200000000000001,"5min":0.191,"15min":0.191},"max":{"1min":0.433,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.216},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.28999999999999998,"15min":0.29799999999999999},"min":{"1min":0.17499999999999999,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":0.40300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.24299999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051637,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:31:35 2023","interfaces":[{"interface":"back","average":{"1min":0.26100000000000001,"5min":0.25700000000000001,"15min":0.251},"min":{"1min":0.19,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.35899999999999999,"5min":0.41599999999999998,"15min":0.41599999999999998},"last":0.221},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.27400000000000002,"15min":0.26700000000000002},"min":{"1min":0.20200000000000001,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.374,"5min":0.42899999999999999,"15min":0.42899999999999999},"last":0.24099999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:32:01.906+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:32:02.388+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:32:02.794 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:32:22.326 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":233,"stamp":"2023-03-25T04:32:22.153289+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002545"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:33.309748+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:29:33.309748+0000","last_peered":"2023-03-25T04:29:33.309748+0000","last_clean":"2023-03-25T04:29:33.309748+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:29:33.309748+0000","last_undegraded":"2023-03-25T04:29:33.309748+0000","last_fullsized":"2023-03-25T04:29:33.309748+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:33.309729+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:29:33.309729+0000","last_peered":"2023-03-25T04:29:33.309729+0000","last_clean":"2023-03-25T04:29:33.309729+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:29:33.309729+0000","last_undegraded":"2023-03-25T04:29:33.309729+0000","last_fullsized":"2023-03-25T04:29:33.309729+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:23.308607+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:29:23.308607+0000","last_peered":"2023-03-25T04:29:23.308607+0000","last_clean":"2023-03-25T04:29:23.308607+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:29:23.308607+0000","last_undegraded":"2023-03-25T04:29:23.308607+0000","last_fullsized":"2023-03-25T04:29:23.308607+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":28,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:20.554710+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:29:20.554710+0000","last_peered":"2023-03-25T04:29:20.554710+0000","last_clean":"2023-03-25T04:29:20.554710+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:29:20.554710+0000","last_undegraded":"2023-03-25T04:29:20.554710+0000","last_fullsized":"2023-03-25T04:29:20.554710+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309216+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:29:28.309216+0000","last_peered":"2023-03-25T04:29:28.309216+0000","last_clean":"2023-03-25T04:29:28.309216+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:29:28.309216+0000","last_undegraded":"2023-03-25T04:29:28.309216+0000","last_fullsized":"2023-03-25T04:29:28.309216+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309233+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:29:28.309233+0000","last_peered":"2023-03-25T04:29:28.309233+0000","last_clean":"2023-03-25T04:29:28.309233+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:29:28.309233+0000","last_undegraded":"2023-03-25T04:29:28.309233+0000","last_fullsized":"2023-03-25T04:29:28.309233+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117055,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:31:42 2023","interfaces":[{"interface":"back","average":{"1min":0.27500000000000002,"5min":0.28399999999999997,"15min":0.29399999999999998},"min":{"1min":0.19800000000000001,"5min":0.191,"15min":0.191},"max":{"1min":0.36899999999999999,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.32300000000000001},{"interface":"front","average":{"1min":0.27900000000000003,"5min":0.28499999999999998,"15min":0.29599999999999999},"min":{"1min":0.216,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":0.40799999999999997,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.27100000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051646,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:31:35 2023","interfaces":[{"interface":"back","average":{"1min":0.26100000000000001,"5min":0.25700000000000001,"15min":0.251},"min":{"1min":0.19,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.35899999999999999,"5min":0.41599999999999998,"15min":0.41599999999999998},"last":0.25600000000000001},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.27400000000000002,"15min":0.26700000000000002},"min":{"1min":0.20200000000000001,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.374,"5min":0.42899999999999999,"15min":0.42899999999999999},"last":0.29199999999999998}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:32:22.327 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean
, 2023-03-25T04:32:22.327 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}

Ignoring malparsed backtrace: 2023-03-25T04:32:39.836 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:32:39.849 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":241,"stamp":"2023-03-25T04:32:38.156312+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002346"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:33.309748+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:29:33.309748+0000","last_peered":"2023-03-25T04:29:33.309748+0000","last_clean":"2023-03-25T04:29:33.309748+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:29:33.309748+0000","last_undegraded":"2023-03-25T04:29:33.309748+0000","last_fullsized":"2023-03-25T04:29:33.309748+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:33.309729+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:29:33.309729+0000","last_peered":"2023-03-25T04:29:33.309729+0000","last_clean":"2023-03-25T04:29:33.309729+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:29:33.309729+0000","last_undegraded":"2023-03-25T04:29:33.309729+0000","last_fullsized":"2023-03-25T04:29:33.309729+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:23.308607+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:29:23.308607+0000","last_peered":"2023-03-25T04:29:23.308607+0000","last_clean":"2023-03-25T04:29:23.308607+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:29:23.308607+0000","last_undegraded":"2023-03-25T04:29:23.308607+0000","last_fullsized":"2023-03-25T04:29:23.308607+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":28,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:20.554710+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:29:20.554710+0000","last_peered":"2023-03-25T04:29:20.554710+0000","last_clean":"2023-03-25T04:29:20.554710+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:29:20.554710+0000","last_undegraded":"2023-03-25T04:29:20.554710+0000","last_fullsized":"2023-03-25T04:29:20.554710+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309216+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:29:28.309216+0000","last_peered":"2023-03-25T04:29:28.309216+0000","last_clean":"2023-03-25T04:29:28.309216+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:29:28.309216+0000","last_undegraded":"2023-03-25T04:29:28.309216+0000","last_fullsized":"2023-03-25T04:29:28.309216+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309233+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:29:28.309233+0000","last_peered":"2023-03-25T04:29:28.309233+0000","last_clean":"2023-03-25T04:29:28.309233+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:29:28.309233+0000","last_undegraded":"2023-03-25T04:29:28.309233+0000","last_fullsized":"2023-03-25T04:29:28.309233+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117058,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:31:42 2023","interfaces":[{"interface":"back","average":{"1min":0.27500000000000002,"5min":0.28399999999999997,"15min":0.29399999999999998},"min":{"1min":0.19800000000000001,"5min":0.191,"15min":0.191},"max":{"1min":0.36899999999999999,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.30599999999999999},{"interface":"front","average":{"1min":0.27900000000000003,"5min":0.28499999999999998,"15min":0.29599999999999999},"min":{"1min":0.216,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":0.40799999999999997,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.32500000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051649,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:31:35 2023","interfaces":[{"interface":"back","average":{"1min":0.26100000000000001,"5min":0.25700000000000001,"15min":0.251},"min":{"1min":0.19,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.35899999999999999,"5min":0.41599999999999998,"15min":0.41599999999999998},"last":0.24199999999999999},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.27400000000000002,"15min":0.26700000000000002},"min":{"1min":0.20200000000000001,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.374,"5min":0.42899999999999999,"15min":0.42899999999999999},"last":0.22600000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:32:54.944+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:32:55.469+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:32:55.927+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)

Ignoring malparsed backtrace: 2023-03-25T04:33:13.958 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean
, 2023-03-25T04:33:13.959 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}
, 2023-03-25T04:33:13.959 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json

Ignoring malparsed backtrace: 2023-03-25T04:33:31.199 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":267,"stamp":"2023-03-25T04:33:30.166660+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002269"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:33.309748+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:29:33.309748+0000","last_peered":"2023-03-25T04:29:33.309748+0000","last_clean":"2023-03-25T04:29:33.309748+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:29:33.309748+0000","last_undegraded":"2023-03-25T04:29:33.309748+0000","last_fullsized":"2023-03-25T04:29:33.309748+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:33.309729+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:29:33.309729+0000","last_peered":"2023-03-25T04:29:33.309729+0000","last_clean":"2023-03-25T04:29:33.309729+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:29:33.309729+0000","last_undegraded":"2023-03-25T04:29:33.309729+0000","last_fullsized":"2023-03-25T04:29:33.309729+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":28,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:31:23.320894+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:31:23.320894+0000","last_peered":"2023-03-25T04:31:23.320894+0000","last_clean":"2023-03-25T04:31:23.320894+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:31:23.320894+0000","last_undegraded":"2023-03-25T04:31:23.320894+0000","last_fullsized":"2023-03-25T04:31:23.320894+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":28,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:20.554710+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:29:20.554710+0000","last_peered":"2023-03-25T04:29:20.554710+0000","last_clean":"2023-03-25T04:29:20.554710+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:29:20.554710+0000","last_undegraded":"2023-03-25T04:29:20.554710+0000","last_fullsized":"2023-03-25T04:29:20.554710+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309216+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:29:28.309216+0000","last_peered":"2023-03-25T04:29:28.309216+0000","last_clean":"2023-03-25T04:29:28.309216+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:29:28.309216+0000","last_undegraded":"2023-03-25T04:29:28.309216+0000","last_fullsized":"2023-03-25T04:29:28.309216+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309233+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:29:28.309233+0000","last_peered":"2023-03-25T04:29:28.309233+0000","last_clean":"2023-03-25T04:29:28.309233+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:29:28.309233+0000","last_undegraded":"2023-03-25T04:29:28.309233+0000","last_fullsized":"2023-03-25T04:29:28.309233+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117068,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:32:46 2023","interfaces":[{"interface":"back","average":{"1min":0.27000000000000002,"5min":0.27900000000000003,"15min":0.29199999999999998},"min":{"1min":0.17899999999999999,"5min":0.17899999999999999,"15min":0.17899999999999999},"max":{"1min":0.35599999999999998,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.28799999999999998},{"interface":"front","average":{"1min":0.26800000000000002,"5min":0.27800000000000002,"15min":0.29399999999999998},"min":{"1min":0.19900000000000001,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":0.377,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.315}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051660,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:32:36 2023","interfaces":[{"interface":"back","average":{"1min":0.248,"5min":0.25700000000000001,"15min":0.251},"min":{"1min":0.17000000000000001,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"last":0.22700000000000001},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.27600000000000002,"15min":0.26800000000000002},"min":{"1min":0.192,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"last":0.20399999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:33:31.201 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:33:31.400+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)

Ignoring malparsed backtrace: 2023-03-25T04:33:47.531+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:33:47.951+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:33:48.505+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)

Ignoring malparsed backtrace: 2023-03-25T04:34:06.104 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json
, 2023-03-25T04:34:06.118+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:34:06.497 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:34:23.399 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:34:23.665+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:34:23.712 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:34:40.195+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:34:40.523+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:34:40.676 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace: 2023-03-25T04:35:03.381+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:35:04.139+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:35:04.391+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)

Ignoring malparsed backtrace: 2023-03-25T04:35:22.738 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean
, 2023-03-25T04:35:22.739 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}
, 2023-03-25T04:35:22.739 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json

Ignoring malparsed backtrace: 2023-03-25T04:35:41.393+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:35:42.352+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:35:42.376+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:35:57.384 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":340,"stamp":"2023-03-25T04:35:56.194992+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.001930"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":36,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:31:43.322894+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:31:43.322894+0000","last_peered":"2023-03-25T04:31:43.322894+0000","last_clean":"2023-03-25T04:31:43.322894+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:31:43.322894+0000","last_undegraded":"2023-03-25T04:31:43.322894+0000","last_fullsized":"2023-03-25T04:31:43.322894+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":28,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:31:43.322876+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:31:43.322876+0000","last_peered":"2023-03-25T04:31:43.322876+0000","last_clean":"2023-03-25T04:31:43.322876+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:31:43.322876+0000","last_undegraded":"2023-03-25T04:31:43.322876+0000","last_fullsized":"2023-03-25T04:31:43.322876+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:23.331998+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:33:23.331998+0000","last_peered":"2023-03-25T04:33:23.331998+0000","last_clean":"2023-03-25T04:33:23.331998+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:33:23.331998+0000","last_undegraded":"2023-03-25T04:33:23.331998+0000","last_fullsized":"2023-03-25T04:33:23.331998+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:30.576723+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:33:30.576723+0000","last_peered":"2023-03-25T04:33:30.576723+0000","last_clean":"2023-03-25T04:33:30.576723+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:33:30.576723+0000","last_undegraded":"2023-03-25T04:33:30.576723+0000","last_fullsized":"2023-03-25T04:33:30.576723+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":37,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:38.333447+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:33:38.333447+0000","last_peered":"2023-03-25T04:33:38.333447+0000","last_clean":"2023-03-25T04:33:38.333447+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:33:38.333447+0000","last_undegraded":"2023-03-25T04:33:38.333447+0000","last_fullsized":"2023-03-25T04:33:38.333447+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:38.333467+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:33:38.333467+0000","last_peered":"2023-03-25T04:33:38.333467+0000","last_clean":"2023-03-25T04:33:38.333467+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:33:38.333467+0000","last_undegraded":"2023-03-25T04:33:38.333467+0000","last_fullsized":"2023-03-25T04:33:38.333467+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117098,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:35:49 2023","interfaces":[{"interface":"back","average":{"1min":0.255,"5min":0.26000000000000001,"15min":0.28199999999999997},"min":{"1min":0.17899999999999999,"5min":0.16,"15min":0.16},"max":{"1min":0.36399999999999999,"5min":0.42599999999999999,"15min":0.45100000000000001},"last":0.26100000000000001},{"interface":"front","average":{"1min":0.26000000000000001,"5min":0.26700000000000002,"15min":0.28599999999999998},"min":{"1min":0.188,"5min":0.17999999999999999,"15min":0.17499999999999999},"max":{"1min":0.39100000000000001,"5min":0.45200000000000001,"15min":0.46300000000000002},"last":0.28399999999999997}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051689,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:35:42 2023","interfaces":[{"interface":"back","average":{"1min":0.29399999999999998,"5min":0.27200000000000002,"15min":0.25800000000000001},"min":{"1min":0.17299999999999999,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.41099999999999998,"5min":0.42699999999999999,"15min":0.42699999999999999},"last":0.20399999999999999},{"interface":"front","average":{"1min":0.30299999999999999,"5min":0.28399999999999997,"15min":0.27300000000000002},"min":{"1min":0.19700000000000001,"5min":0.192,"15min":0.16600000000000001},"max":{"1min":0.43099999999999999,"5min":0.45800000000000002,"15min":0.45800000000000002},"last":0.22}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:35:57.624+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)

Ignoring malparsed backtrace: 2023-03-25T04:36:17.226 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:36:17.239 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":350,"stamp":"2023-03-25T04:36:16.198866+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002148"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":37,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:53.335026+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:33:53.335026+0000","last_peered":"2023-03-25T04:33:53.335026+0000","last_clean":"2023-03-25T04:33:53.335026+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:33:53.335026+0000","last_undegraded":"2023-03-25T04:33:53.335026+0000","last_fullsized":"2023-03-25T04:33:53.335026+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:53.335009+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:33:53.335009+0000","last_peered":"2023-03-25T04:33:53.335009+0000","last_clean":"2023-03-25T04:33:53.335009+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:33:53.335009+0000","last_undegraded":"2023-03-25T04:33:53.335009+0000","last_fullsized":"2023-03-25T04:33:53.335009+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:23.331998+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:33:23.331998+0000","last_peered":"2023-03-25T04:33:23.331998+0000","last_clean":"2023-03-25T04:33:23.331998+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:33:23.331998+0000","last_undegraded":"2023-03-25T04:33:23.331998+0000","last_fullsized":"2023-03-25T04:33:23.331998+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:30.576723+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:33:30.576723+0000","last_peered":"2023-03-25T04:33:30.576723+0000","last_clean":"2023-03-25T04:33:30.576723+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:33:30.576723+0000","last_undegraded":"2023-03-25T04:33:30.576723+0000","last_fullsized":"2023-03-25T04:33:30.576723+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":37,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:38.333447+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:33:38.333447+0000","last_peered":"2023-03-25T04:33:38.333447+0000","last_clean":"2023-03-25T04:33:38.333447+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:33:38.333447+0000","last_undegraded":"2023-03-25T04:33:38.333447+0000","last_fullsized":"2023-03-25T04:33:38.333447+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:38.333467+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:33:38.333467+0000","last_peered":"2023-03-25T04:33:38.333467+0000","last_clean":"2023-03-25T04:33:38.333467+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:33:38.333467+0000","last_undegraded":"2023-03-25T04:33:38.333467+0000","last_fullsized":"2023-03-25T04:33:38.333467+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117102,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:35:49 2023","interfaces":[{"interface":"back","average":{"1min":0.255,"5min":0.26000000000000001,"15min":0.28199999999999997},"min":{"1min":0.17899999999999999,"5min":0.16,"15min":0.16},"max":{"1min":0.36399999999999999,"5min":0.42599999999999999,"15min":0.45100000000000001},"last":0.28399999999999997},{"interface":"front","average":{"1min":0.26000000000000001,"5min":0.26700000000000002,"15min":0.28599999999999998},"min":{"1min":0.188,"5min":0.17999999999999999,"15min":0.17499999999999999},"max":{"1min":0.39100000000000001,"5min":0.45200000000000001,"15min":0.46300000000000002},"last":0.30299999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051693,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:35:42 2023","interfaces":[{"interface":"back","average":{"1min":0.29399999999999998,"5min":0.27200000000000002,"15min":0.25800000000000001},"min":{"1min":0.17299999999999999,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.41099999999999998,"5min":0.42699999999999999,"15min":0.42699999999999999},"last":0.35099999999999998},{"interface":"front","average":{"1min":0.30299999999999999,"5min":0.28399999999999997,"15min":0.27300000000000002},"min":{"1min":0.19700000000000001,"5min":0.192,"15min":0.16600000000000001},"max":{"1min":0.43099999999999999,"5min":0.45800000000000002,"15min":0.45800000000000002},"last":0.32900000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:36:34.556+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:36:34.717+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:36:35.583+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)

Ignoring malparsed backtrace: 2023-03-25T04:36:52.680 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json
, 2023-03-25T04:36:52.701+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:36:53.067 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:37:09.963 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:37:10.273 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all

Ignoring malparsed backtrace: 2023-03-25T04:37:26.856+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:37:27.288 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:37:27.538+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:37:41.903 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":392,"stamp":"2023-03-25T04:37:40.213373+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002012"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":37,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:53.335026+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:33:53.335026+0000","last_peered":"2023-03-25T04:33:53.335026+0000","last_clean":"2023-03-25T04:33:53.335026+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:33:53.335026+0000","last_undegraded":"2023-03-25T04:33:53.335026+0000","last_fullsized":"2023-03-25T04:33:53.335026+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:53.335009+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:33:53.335009+0000","last_peered":"2023-03-25T04:33:53.335009+0000","last_clean":"2023-03-25T04:33:53.335009+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:33:53.335009+0000","last_undegraded":"2023-03-25T04:33:53.335009+0000","last_fullsized":"2023-03-25T04:33:53.335009+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:23.343585+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:35:23.343585+0000","last_peered":"2023-03-25T04:35:23.343585+0000","last_clean":"2023-03-25T04:35:23.343585+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:35:23.343585+0000","last_undegraded":"2023-03-25T04:35:23.343585+0000","last_fullsized":"2023-03-25T04:35:23.343585+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:30.576723+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:33:30.576723+0000","last_peered":"2023-03-25T04:33:30.576723+0000","last_clean":"2023-03-25T04:33:30.576723+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:33:30.576723+0000","last_undegraded":"2023-03-25T04:33:30.576723+0000","last_fullsized":"2023-03-25T04:33:30.576723+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":37,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:38.333447+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:33:38.333447+0000","last_peered":"2023-03-25T04:33:38.333447+0000","last_clean":"2023-03-25T04:33:38.333447+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:33:38.333447+0000","last_undegraded":"2023-03-25T04:33:38.333447+0000","last_fullsized":"2023-03-25T04:33:38.333447+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:38.333467+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:33:38.333467+0000","last_peered":"2023-03-25T04:33:38.333467+0000","last_clean":"2023-03-25T04:33:38.333467+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:33:38.333467+0000","last_undegraded":"2023-03-25T04:33:38.333467+0000","last_fullsized":"2023-03-25T04:33:38.333467+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117119,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:36:50 2023","interfaces":[{"interface":"back","average":{"1min":0.254,"5min":0.25600000000000001,"15min":0.27900000000000003},"min":{"1min":0.20499999999999999,"5min":0.16,"15min":0.16},"max":{"1min":0.371,"5min":0.42599999999999999,"15min":0.45100000000000001},"last":0.23000000000000001},{"interface":"front","average":{"1min":0.26800000000000002,"5min":0.26500000000000001,"15min":0.28399999999999997},"min":{"1min":0.19700000000000001,"5min":0.17999999999999999,"15min":0.17499999999999999},"max":{"1min":0.34699999999999998,"5min":0.45200000000000001,"15min":0.46300000000000002},"last":0.20899999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051710,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:36:44 2023","interfaces":[{"interface":"back","average":{"1min":0.28399999999999997,"5min":0.27700000000000002,"15min":0.26100000000000001},"min":{"1min":0.192,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.40899999999999997,"5min":0.42699999999999999,"15min":0.42699999999999999},"last":0.39700000000000002},{"interface":"front","average":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.27500000000000002},"min":{"1min":0.182,"5min":0.182,"15min":0.16600000000000001},"max":{"1min":0.439,"5min":0.45800000000000002,"15min":0.45800000000000002},"last":0.439}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:37:42.400+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)

Ignoring malparsed backtrace: 2023-03-25T04:38:04.766+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:38:04.922 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:38:05.246 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:38:19.605 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":411,"stamp":"2023-03-25T04:38:18.220966+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002368"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":38,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:36:03.347070+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:36:03.347070+0000","last_peered":"2023-03-25T04:36:03.347070+0000","last_clean":"2023-03-25T04:36:03.347070+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:36:03.347070+0000","last_undegraded":"2023-03-25T04:36:03.347070+0000","last_fullsized":"2023-03-25T04:36:03.347070+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:36:03.347025+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:36:03.347025+0000","last_peered":"2023-03-25T04:36:03.347025+0000","last_clean":"2023-03-25T04:36:03.347025+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:36:03.347025+0000","last_undegraded":"2023-03-25T04:36:03.347025+0000","last_fullsized":"2023-03-25T04:36:03.347025+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:23.343585+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:35:23.343585+0000","last_peered":"2023-03-25T04:35:23.343585+0000","last_clean":"2023-03-25T04:35:23.343585+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:35:23.343585+0000","last_undegraded":"2023-03-25T04:35:23.343585+0000","last_fullsized":"2023-03-25T04:35:23.343585+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:35.587025+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:35:35.587025+0000","last_peered":"2023-03-25T04:35:35.587025+0000","last_clean":"2023-03-25T04:35:35.587025+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:35:35.587025+0000","last_undegraded":"2023-03-25T04:35:35.587025+0000","last_fullsized":"2023-03-25T04:35:35.587025+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":38,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:43.345308+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:35:43.345308+0000","last_peered":"2023-03-25T04:35:43.345308+0000","last_clean":"2023-03-25T04:35:43.345308+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:35:43.345308+0000","last_undegraded":"2023-03-25T04:35:43.345308+0000","last_fullsized":"2023-03-25T04:35:43.345308+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:43.345324+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:35:43.345324+0000","last_peered":"2023-03-25T04:35:43.345324+0000","last_clean":"2023-03-25T04:35:43.345324+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:35:43.345324+0000","last_undegraded":"2023-03-25T04:35:43.345324+0000","last_fullsized":"2023-03-25T04:35:43.345324+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117127,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:37:53 2023","interfaces":[{"interface":"back","average":{"1min":0.25800000000000001,"5min":0.253,"15min":0.27700000000000002},"min":{"1min":0.184,"5min":0.16,"15min":0.16},"max":{"1min":0.44600000000000001,"5min":0.44600000000000001,"15min":0.45100000000000001},"last":0.35199999999999998},{"interface":"front","average":{"1min":0.26900000000000002,"5min":0.26500000000000001,"15min":0.28199999999999997},"min":{"1min":0.192,"5min":0.17999999999999999,"15min":0.17499999999999999},"max":{"1min":0.46400000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"last":0.33200000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051717,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:37:44 2023","interfaces":[{"interface":"back","average":{"1min":0.27300000000000002,"5min":0.28199999999999997,"15min":0.26200000000000001},"min":{"1min":0.17899999999999999,"5min":0.17299999999999999,"15min":0.17000000000000001},"max":{"1min":0.41799999999999998,"5min":0.42099999999999999,"15min":0.42699999999999999},"last":0.28699999999999998},{"interface":"front","average":{"1min":0.29399999999999998,"5min":0.29099999999999998,"15min":0.27700000000000002},"min":{"1min":0.19900000000000001,"5min":0.182,"15min":0.16600000000000001},"max":{"1min":0.441,"5min":0.441,"15min":0.45800000000000002},"last":0.26300000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:38:19.718+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:38:20.315+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)

Ignoring malparsed backtrace: 2023-03-25T04:38:44.283 INFO:teuthology.orchestra.run.smithi070.stdout:
, 2023-03-25T04:38:44.283 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":781,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":16,"modified":"2023-03-25T04:38:40.225321+0000","services":{}},"progress_events":{}}
, 2023-03-25T04:38:44.296 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now

Ignoring malparsed backtrace: 2023-03-25T04:39:02.477+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:39:02.578+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:39:03.524+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:39:20.359 INFO:teuthology.orchestra.run.smithi070.stdout:
, 2023-03-25T04:39:20.359 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":818,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":16,"modified":"2023-03-25T04:38:40.225321+0000","services":{}},"progress_events":{}}
, 2023-03-25T04:39:20.372 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now

Ignoring malparsed backtrace: 2023-03-25T04:39:37.527 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:39:37.541 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":450,"stamp":"2023-03-25T04:39:36.235332+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002086"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":38,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:36:03.347070+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:36:03.347070+0000","last_peered":"2023-03-25T04:36:03.347070+0000","last_clean":"2023-03-25T04:36:03.347070+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:36:03.347070+0000","last_undegraded":"2023-03-25T04:36:03.347070+0000","last_fullsized":"2023-03-25T04:36:03.347070+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:36:03.347025+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:36:03.347025+0000","last_peered":"2023-03-25T04:36:03.347025+0000","last_clean":"2023-03-25T04:36:03.347025+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:36:03.347025+0000","last_undegraded":"2023-03-25T04:36:03.347025+0000","last_fullsized":"2023-03-25T04:36:03.347025+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:23.353778+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:37:23.353778+0000","last_peered":"2023-03-25T04:37:23.353778+0000","last_clean":"2023-03-25T04:37:23.353778+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:37:23.353778+0000","last_undegraded":"2023-03-25T04:37:23.353778+0000","last_fullsized":"2023-03-25T04:37:23.353778+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:35.587025+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:35:35.587025+0000","last_peered":"2023-03-25T04:35:35.587025+0000","last_clean":"2023-03-25T04:35:35.587025+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:35:35.587025+0000","last_undegraded":"2023-03-25T04:35:35.587025+0000","last_fullsized":"2023-03-25T04:35:35.587025+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":38,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:43.345308+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:35:43.345308+0000","last_peered":"2023-03-25T04:35:43.345308+0000","last_clean":"2023-03-25T04:35:43.345308+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:35:43.345308+0000","last_undegraded":"2023-03-25T04:35:43.345308+0000","last_fullsized":"2023-03-25T04:35:43.345308+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:43.345324+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:35:43.345324+0000","last_peered":"2023-03-25T04:35:43.345324+0000","last_clean":"2023-03-25T04:35:43.345324+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:35:43.345324+0000","last_undegraded":"2023-03-25T04:35:43.345324+0000","last_fullsized":"2023-03-25T04:35:43.345324+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117143,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:38:54 2023","interfaces":[{"interface":"back","average":{"1min":0.25800000000000001,"5min":0.25700000000000001,"15min":0.27400000000000002},"min":{"1min":0.192,"5min":0.17399999999999999,"15min":0.16},"max":{"1min":0.39400000000000002,"5min":0.44600000000000001,"15min":0.45100000000000001},"last":0.26500000000000001},{"interface":"front","average":{"1min":0.25900000000000001,"5min":0.26700000000000002,"15min":0.27900000000000003},"min":{"1min":0.19500000000000001,"5min":0.188,"15min":0.17499999999999999},"max":{"1min":0.41499999999999998,"5min":0.46400000000000002,"15min":0.46400000000000002},"last":0.29099999999999998}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051733,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:38:45 2023","interfaces":[{"interface":"back","average":{"1min":0.27300000000000002,"5min":0.28100000000000003,"15min":0.26400000000000001},"min":{"1min":0.19400000000000001,"5min":0.17299999999999999,"15min":0.17000000000000001},"max":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"last":0.27300000000000002},{"interface":"front","average":{"1min":0.28799999999999998,"5min":0.29099999999999998,"15min":0.27800000000000002},"min":{"1min":0.19800000000000001,"5min":0.182,"15min":0.16600000000000001},"max":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"last":0.308}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:39:54.416+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:39:54.455 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:39:54.780 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:40:09.288+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:40:09.389+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:40:10.267+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)

Ignoring malparsed backtrace: 2023-03-25T04:40:28.984 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":476,"stamp":"2023-03-25T04:40:28.245911+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002693"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":39,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:38:13.358968+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:38:13.358968+0000","last_peered":"2023-03-25T04:38:13.358968+0000","last_clean":"2023-03-25T04:38:13.358968+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:38:13.358968+0000","last_undegraded":"2023-03-25T04:38:13.358968+0000","last_fullsized":"2023-03-25T04:38:13.358968+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:38:13.358922+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:38:13.358922+0000","last_peered":"2023-03-25T04:38:13.358922+0000","last_clean":"2023-03-25T04:38:13.358922+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:38:13.358922+0000","last_undegraded":"2023-03-25T04:38:13.358922+0000","last_fullsized":"2023-03-25T04:38:13.358922+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:23.353778+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:37:23.353778+0000","last_peered":"2023-03-25T04:37:23.353778+0000","last_clean":"2023-03-25T04:37:23.353778+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:37:23.353778+0000","last_undegraded":"2023-03-25T04:37:23.353778+0000","last_fullsized":"2023-03-25T04:37:23.353778+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:40.598579+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:37:40.598579+0000","last_peered":"2023-03-25T04:37:40.598579+0000","last_clean":"2023-03-25T04:37:40.598579+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:37:40.598579+0000","last_undegraded":"2023-03-25T04:37:40.598579+0000","last_fullsized":"2023-03-25T04:37:40.598579+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":39,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:48.356344+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:37:48.356344+0000","last_peered":"2023-03-25T04:37:48.356344+0000","last_clean":"2023-03-25T04:37:48.356344+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:37:48.356344+0000","last_undegraded":"2023-03-25T04:37:48.356344+0000","last_fullsized":"2023-03-25T04:37:48.356344+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:48.356365+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:37:48.356365+0000","last_peered":"2023-03-25T04:37:48.356365+0000","last_clean":"2023-03-25T04:37:48.356365+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:37:48.356365+0000","last_undegraded":"2023-03-25T04:37:48.356365+0000","last_fullsized":"2023-03-25T04:37:48.356365+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117153,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:39:54 2023","interfaces":[{"interface":"back","average":{"1min":0.25900000000000001,"5min":0.25600000000000001,"15min":0.27100000000000002},"min":{"1min":0.188,"5min":0.17899999999999999,"15min":0.16},"max":{"1min":0.41999999999999998,"5min":0.44600000000000001,"15min":0.45100000000000001},"last":0.255},{"interface":"front","average":{"1min":0.27500000000000002,"5min":0.26600000000000001,"15min":0.27700000000000002},"min":{"1min":0.17699999999999999,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.45600000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"last":0.26800000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051743,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:39:47 2023","interfaces":[{"interface":"back","average":{"1min":0.26300000000000001,"5min":0.27700000000000002,"15min":0.26500000000000001},"min":{"1min":0.187,"5min":0.17299999999999999,"15min":0.17000000000000001},"max":{"1min":0.433,"5min":0.433,"15min":0.433},"last":0.36599999999999999},{"interface":"front","average":{"1min":0.27100000000000002,"5min":0.28799999999999998,"15min":0.27900000000000003},"min":{"1min":0.20100000000000001,"5min":0.182,"15min":0.16600000000000001},"max":{"1min":0.48599999999999999,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.34999999999999998}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:40:28.986 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean
, 2023-03-25T04:40:28.986 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}

Ignoring malparsed backtrace: 2023-03-25T04:40:47.229+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:40:47.503+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:40:48.251+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)

Ignoring malparsed backtrace: 2023-03-25T04:41:03.267 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:41:03.578 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all

Ignoring malparsed backtrace: 2023-03-25T04:41:21.347 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":502,"stamp":"2023-03-25T04:41:20.256204+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002342"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":39,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:38:13.358968+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:38:13.358968+0000","last_peered":"2023-03-25T04:38:13.358968+0000","last_clean":"2023-03-25T04:38:13.358968+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:38:13.358968+0000","last_undegraded":"2023-03-25T04:38:13.358968+0000","last_fullsized":"2023-03-25T04:38:13.358968+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:38:13.358922+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:38:13.358922+0000","last_peered":"2023-03-25T04:38:13.358922+0000","last_clean":"2023-03-25T04:38:13.358922+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:38:13.358922+0000","last_undegraded":"2023-03-25T04:38:13.358922+0000","last_fullsized":"2023-03-25T04:38:13.358922+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:23.353778+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:37:23.353778+0000","last_peered":"2023-03-25T04:37:23.353778+0000","last_clean":"2023-03-25T04:37:23.353778+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:37:23.353778+0000","last_undegraded":"2023-03-25T04:37:23.353778+0000","last_fullsized":"2023-03-25T04:37:23.353778+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:40.598579+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:37:40.598579+0000","last_peered":"2023-03-25T04:37:40.598579+0000","last_clean":"2023-03-25T04:37:40.598579+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:37:40.598579+0000","last_undegraded":"2023-03-25T04:37:40.598579+0000","last_fullsized":"2023-03-25T04:37:40.598579+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":39,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:48.356344+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:37:48.356344+0000","last_peered":"2023-03-25T04:37:48.356344+0000","last_clean":"2023-03-25T04:37:48.356344+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:37:48.356344+0000","last_undegraded":"2023-03-25T04:37:48.356344+0000","last_fullsized":"2023-03-25T04:37:48.356344+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:48.356365+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:37:48.356365+0000","last_peered":"2023-03-25T04:37:48.356365+0000","last_clean":"2023-03-25T04:37:48.356365+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:37:48.356365+0000","last_undegraded":"2023-03-25T04:37:48.356365+0000","last_fullsized":"2023-03-25T04:37:48.356365+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117163,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:40:56 2023","interfaces":[{"interface":"back","average":{"1min":0.28799999999999998,"5min":0.26300000000000001,"15min":0.27100000000000002},"min":{"1min":0.188,"5min":0.184,"15min":0.16},"max":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.17499999999999999},{"interface":"front","average":{"1min":0.29599999999999999,"5min":0.27300000000000002,"15min":0.27700000000000002},"min":{"1min":0.20599999999999999,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.43099999999999999,"5min":0.46400000000000002,"15min":0.46400000000000002},"last":0.20699999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051754,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:40:52 2023","interfaces":[{"interface":"back","average":{"1min":0.28599999999999998,"5min":0.27500000000000002,"15min":0.26700000000000002},"min":{"1min":0.17899999999999999,"5min":0.17899999999999999,"15min":0.17000000000000001},"max":{"1min":0.44,"5min":0.44,"15min":0.44},"last":0.23699999999999999},{"interface":"front","average":{"1min":0.29799999999999999,"5min":0.28699999999999998,"15min":0.28100000000000003},"min":{"1min":0.20300000000000001,"5min":0.182,"15min":0.16600000000000001},"max":{"1min":0.47599999999999998,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.28000000000000003}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:41:21.349 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean
, 2023-03-25T04:41:21.349 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}

Ignoring malparsed backtrace: 2023-03-25T04:41:39.867+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:41:40.383+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:41:40.905+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:41:58.219 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json
, 2023-03-25T04:41:58.319+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:41:58.644 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:42:18.193 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":530,"stamp":"2023-03-25T04:42:16.266659+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002356"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":39,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:38:13.358968+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:38:13.358968+0000","last_peered":"2023-03-25T04:38:13.358968+0000","last_clean":"2023-03-25T04:38:13.358968+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:38:13.358968+0000","last_undegraded":"2023-03-25T04:38:13.358968+0000","last_fullsized":"2023-03-25T04:38:13.358968+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:38:13.358922+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:38:13.358922+0000","last_peered":"2023-03-25T04:38:13.358922+0000","last_clean":"2023-03-25T04:38:13.358922+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:38:13.358922+0000","last_undegraded":"2023-03-25T04:38:13.358922+0000","last_fullsized":"2023-03-25T04:38:13.358922+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:23.364593+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:39:23.364593+0000","last_peered":"2023-03-25T04:39:23.364593+0000","last_clean":"2023-03-25T04:39:23.364593+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:39:23.364593+0000","last_undegraded":"2023-03-25T04:39:23.364593+0000","last_fullsized":"2023-03-25T04:39:23.364593+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:45.608475+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:39:45.608475+0000","last_peered":"2023-03-25T04:39:45.608475+0000","last_clean":"2023-03-25T04:39:45.608475+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:39:45.608475+0000","last_undegraded":"2023-03-25T04:39:45.608475+0000","last_fullsized":"2023-03-25T04:39:45.608475+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":40,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:53.367398+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:39:53.367398+0000","last_peered":"2023-03-25T04:39:53.367398+0000","last_clean":"2023-03-25T04:39:53.367398+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:39:53.367398+0000","last_undegraded":"2023-03-25T04:39:53.367398+0000","last_fullsized":"2023-03-25T04:39:53.367398+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:53.367424+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:39:53.367424+0000","last_peered":"2023-03-25T04:39:53.367424+0000","last_clean":"2023-03-25T04:39:53.367424+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:39:53.367424+0000","last_undegraded":"2023-03-25T04:39:53.367424+0000","last_fullsized":"2023-03-25T04:39:53.367424+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117175,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:41:58 2023","interfaces":[{"interface":"back","average":{"1min":0.25800000000000001,"5min":0.26400000000000001,"15min":0.26800000000000002},"min":{"1min":0.17499999999999999,"5min":0.17499999999999999,"15min":0.16},"max":{"1min":0.39100000000000001,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.22900000000000001},{"interface":"front","average":{"1min":0.26300000000000001,"5min":0.27200000000000002,"15min":0.27400000000000002},"min":{"1min":0.20599999999999999,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.35699999999999998,"5min":0.46400000000000002,"15min":0.46400000000000002},"last":0.26400000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051765,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:41:52 2023","interfaces":[{"interface":"back","average":{"1min":0.26500000000000001,"5min":0.27200000000000002,"15min":0.26800000000000002},"min":{"1min":0.17000000000000001,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.39200000000000002,"5min":0.44,"15min":0.44},"last":0.23899999999999999},{"interface":"front","average":{"1min":0.28100000000000003,"5min":0.28599999999999998,"15min":0.28199999999999997},"min":{"1min":0.19400000000000001,"5min":0.19400000000000001,"15min":0.16600000000000001},"max":{"1min":0.41399999999999998,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.254}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:42:18.297+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:42:18.958+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:42:35.512 INFO:teuthology.orchestra.run.smithi070.stdout:
, 2023-03-25T04:42:35.512 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":1013,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":17,"modified":"2023-03-25T04:41:54.262744+0000","services":{}},"progress_events":{}}
, 2023-03-25T04:42:35.525 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:42:52.741 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":548,"stamp":"2023-03-25T04:42:52.273601+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002221"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":40,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:40:23.370056+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:40:23.370056+0000","last_peered":"2023-03-25T04:40:23.370056+0000","last_clean":"2023-03-25T04:40:23.370056+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:40:23.370056+0000","last_undegraded":"2023-03-25T04:40:23.370056+0000","last_fullsized":"2023-03-25T04:40:23.370056+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:40:23.370039+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:40:23.370039+0000","last_peered":"2023-03-25T04:40:23.370039+0000","last_clean":"2023-03-25T04:40:23.370039+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:40:23.370039+0000","last_undegraded":"2023-03-25T04:40:23.370039+0000","last_fullsized":"2023-03-25T04:40:23.370039+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:23.364593+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:39:23.364593+0000","last_peered":"2023-03-25T04:39:23.364593+0000","last_clean":"2023-03-25T04:39:23.364593+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:39:23.364593+0000","last_undegraded":"2023-03-25T04:39:23.364593+0000","last_fullsized":"2023-03-25T04:39:23.364593+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:45.608475+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:39:45.608475+0000","last_peered":"2023-03-25T04:39:45.608475+0000","last_clean":"2023-03-25T04:39:45.608475+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:39:45.608475+0000","last_undegraded":"2023-03-25T04:39:45.608475+0000","last_fullsized":"2023-03-25T04:39:45.608475+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":40,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:53.367398+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:39:53.367398+0000","last_peered":"2023-03-25T04:39:53.367398+0000","last_clean":"2023-03-25T04:39:53.367398+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:39:53.367398+0000","last_undegraded":"2023-03-25T04:39:53.367398+0000","last_fullsized":"2023-03-25T04:39:53.367398+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:53.367424+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:39:53.367424+0000","last_peered":"2023-03-25T04:39:53.367424+0000","last_clean":"2023-03-25T04:39:53.367424+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:39:53.367424+0000","last_undegraded":"2023-03-25T04:39:53.367424+0000","last_fullsized":"2023-03-25T04:39:53.367424+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117182,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:41:58 2023","interfaces":[{"interface":"back","average":{"1min":0.25800000000000001,"5min":0.26400000000000001,"15min":0.26800000000000002},"min":{"1min":0.17499999999999999,"5min":0.17499999999999999,"15min":0.16},"max":{"1min":0.39100000000000001,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.19600000000000001},{"interface":"front","average":{"1min":0.26300000000000001,"5min":0.27200000000000002,"15min":0.27400000000000002},"min":{"1min":0.20599999999999999,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.35699999999999998,"5min":0.46400000000000002,"15min":0.46400000000000002},"last":0.219}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051773,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:41:52 2023","interfaces":[{"interface":"back","average":{"1min":0.26500000000000001,"5min":0.27200000000000002,"15min":0.26800000000000002},"min":{"1min":0.17000000000000001,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.39200000000000002,"5min":0.44,"15min":0.44},"last":0.19},{"interface":"front","average":{"1min":0.28100000000000003,"5min":0.28599999999999998,"15min":0.28199999999999997},"min":{"1min":0.19400000000000001,"5min":0.19400000000000001,"15min":0.16600000000000001},"max":{"1min":0.41399999999999998,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.17599999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:42:52.742 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean

Ignoring malparsed backtrace: 2023-03-25T04:43:09.973 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:43:09.986 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":556,"stamp":"2023-03-25T04:43:08.276556+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002359"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":40,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:40:23.370056+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:40:23.370056+0000","last_peered":"2023-03-25T04:40:23.370056+0000","last_clean":"2023-03-25T04:40:23.370056+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:40:23.370056+0000","last_undegraded":"2023-03-25T04:40:23.370056+0000","last_fullsized":"2023-03-25T04:40:23.370056+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:40:23.370039+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:40:23.370039+0000","last_peered":"2023-03-25T04:40:23.370039+0000","last_clean":"2023-03-25T04:40:23.370039+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:40:23.370039+0000","last_undegraded":"2023-03-25T04:40:23.370039+0000","last_fullsized":"2023-03-25T04:40:23.370039+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:23.364593+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:39:23.364593+0000","last_peered":"2023-03-25T04:39:23.364593+0000","last_clean":"2023-03-25T04:39:23.364593+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:39:23.364593+0000","last_undegraded":"2023-03-25T04:39:23.364593+0000","last_fullsized":"2023-03-25T04:39:23.364593+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:45.608475+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:39:45.608475+0000","last_peered":"2023-03-25T04:39:45.608475+0000","last_clean":"2023-03-25T04:39:45.608475+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:39:45.608475+0000","last_undegraded":"2023-03-25T04:39:45.608475+0000","last_fullsized":"2023-03-25T04:39:45.608475+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":40,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:53.367398+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:39:53.367398+0000","last_peered":"2023-03-25T04:39:53.367398+0000","last_clean":"2023-03-25T04:39:53.367398+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:39:53.367398+0000","last_undegraded":"2023-03-25T04:39:53.367398+0000","last_fullsized":"2023-03-25T04:39:53.367398+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:53.367424+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:39:53.367424+0000","last_peered":"2023-03-25T04:39:53.367424+0000","last_clean":"2023-03-25T04:39:53.367424+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:39:53.367424+0000","last_undegraded":"2023-03-25T04:39:53.367424+0000","last_fullsized":"2023-03-25T04:39:53.367424+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117185,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:43:00 2023","interfaces":[{"interface":"back","average":{"1min":0.27400000000000002,"5min":0.26700000000000002,"15min":0.26600000000000001},"min":{"1min":0.19600000000000001,"5min":0.17499999999999999,"15min":0.16},"max":{"1min":0.42999999999999999,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.27800000000000002},{"interface":"front","average":{"1min":0.28199999999999997,"5min":0.27500000000000002,"15min":0.27300000000000002},"min":{"1min":0.185,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.47199999999999998,"5min":0.47199999999999998,"15min":0.47199999999999998},"last":0.247}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051776,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:42:54 2023","interfaces":[{"interface":"back","average":{"1min":0.26300000000000001,"5min":0.27000000000000002,"15min":0.26900000000000002},"min":{"1min":0.17399999999999999,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.438,"5min":0.44,"15min":0.44},"last":0.20100000000000001},{"interface":"front","average":{"1min":0.27400000000000002,"5min":0.28199999999999997,"15min":0.28299999999999997},"min":{"1min":0.17599999999999999,"5min":0.17599999999999999,"15min":0.16600000000000001},"max":{"1min":0.41599999999999998,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.23499999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:43:25.042+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:43:25.411+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:43:26.013+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:43:47.937+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:43:48.534+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:43:48.889+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:44:09.318 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:44:09.331 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":586,"stamp":"2023-03-25T04:44:08.288933+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002697"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":40,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:40:23.370056+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:40:23.370056+0000","last_peered":"2023-03-25T04:40:23.370056+0000","last_clean":"2023-03-25T04:40:23.370056+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:40:23.370056+0000","last_undegraded":"2023-03-25T04:40:23.370056+0000","last_fullsized":"2023-03-25T04:40:23.370056+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:40:23.370039+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:40:23.370039+0000","last_peered":"2023-03-25T04:40:23.370039+0000","last_clean":"2023-03-25T04:40:23.370039+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:40:23.370039+0000","last_undegraded":"2023-03-25T04:40:23.370039+0000","last_fullsized":"2023-03-25T04:40:23.370039+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:23.375106+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:41:23.375106+0000","last_peered":"2023-03-25T04:41:23.375106+0000","last_clean":"2023-03-25T04:41:23.375106+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:41:23.375106+0000","last_undegraded":"2023-03-25T04:41:23.375106+0000","last_fullsized":"2023-03-25T04:41:23.375106+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:50.618930+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:41:50.618930+0000","last_peered":"2023-03-25T04:41:50.618930+0000","last_clean":"2023-03-25T04:41:50.618930+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:41:50.618930+0000","last_undegraded":"2023-03-25T04:41:50.618930+0000","last_fullsized":"2023-03-25T04:41:50.618930+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":41,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:58.377821+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:41:58.377821+0000","last_peered":"2023-03-25T04:41:58.377821+0000","last_clean":"2023-03-25T04:41:58.377821+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:41:58.377821+0000","last_undegraded":"2023-03-25T04:41:58.377821+0000","last_fullsized":"2023-03-25T04:41:58.377821+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:58.377836+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:41:58.377836+0000","last_peered":"2023-03-25T04:41:58.377836+0000","last_clean":"2023-03-25T04:41:58.377836+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:41:58.377836+0000","last_undegraded":"2023-03-25T04:41:58.377836+0000","last_fullsized":"2023-03-25T04:41:58.377836+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117197,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:44:01 2023","interfaces":[{"interface":"back","average":{"1min":0.27200000000000002,"5min":0.27000000000000002,"15min":0.26400000000000001},"min":{"1min":0.20399999999999999,"5min":0.17499999999999999,"15min":0.16},"max":{"1min":0.38400000000000001,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.245},{"interface":"front","average":{"1min":0.27600000000000002,"5min":0.27800000000000002,"15min":0.27100000000000002},"min":{"1min":0.23100000000000001,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.40400000000000003,"5min":0.47199999999999998,"15min":0.47199999999999998},"last":0.27500000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051788,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:43:55 2023","interfaces":[{"interface":"back","average":{"1min":0.26100000000000001,"5min":0.26700000000000002,"15min":0.27000000000000002},"min":{"1min":0.17799999999999999,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.374,"5min":0.44,"15min":0.44},"last":0.30299999999999999},{"interface":"front","average":{"1min":0.28299999999999997,"5min":0.28100000000000003,"15min":0.28399999999999997},"min":{"1min":0.20899999999999999,"5min":0.17599999999999999,"15min":0.16600000000000001},"max":{"1min":0.40100000000000002,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.27800000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:44:28.923 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":1126,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":19,"modified":"2023-03-25T04:43:20.278959+0000","services":{}},"progress_events":{}}
, 2023-03-25T04:44:28.936 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now
, 2023-03-25T04:44:28.937 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace: 2023-03-25T04:44:48.786 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:44:48.800 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":606,"stamp":"2023-03-25T04:44:48.296805+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002182"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":41,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:42:34.382196+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:42:34.382196+0000","last_peered":"2023-03-25T04:42:34.382196+0000","last_clean":"2023-03-25T04:42:34.382196+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:42:34.382196+0000","last_undegraded":"2023-03-25T04:42:34.382196+0000","last_fullsized":"2023-03-25T04:42:34.382196+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:42:29.381744+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:42:29.381744+0000","last_peered":"2023-03-25T04:42:29.381744+0000","last_clean":"2023-03-25T04:42:29.381744+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:42:29.381744+0000","last_undegraded":"2023-03-25T04:42:29.381744+0000","last_fullsized":"2023-03-25T04:42:29.381744+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:23.375106+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:41:23.375106+0000","last_peered":"2023-03-25T04:41:23.375106+0000","last_clean":"2023-03-25T04:41:23.375106+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:41:23.375106+0000","last_undegraded":"2023-03-25T04:41:23.375106+0000","last_fullsized":"2023-03-25T04:41:23.375106+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:50.618930+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:41:50.618930+0000","last_peered":"2023-03-25T04:41:50.618930+0000","last_clean":"2023-03-25T04:41:50.618930+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:41:50.618930+0000","last_undegraded":"2023-03-25T04:41:50.618930+0000","last_fullsized":"2023-03-25T04:41:50.618930+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":41,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:58.377821+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:41:58.377821+0000","last_peered":"2023-03-25T04:41:58.377821+0000","last_clean":"2023-03-25T04:41:58.377821+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:41:58.377821+0000","last_undegraded":"2023-03-25T04:41:58.377821+0000","last_fullsized":"2023-03-25T04:41:58.377821+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:58.377836+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:41:58.377836+0000","last_peered":"2023-03-25T04:41:58.377836+0000","last_clean":"2023-03-25T04:41:58.377836+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:41:58.377836+0000","last_undegraded":"2023-03-25T04:41:58.377836+0000","last_fullsized":"2023-03-25T04:41:58.377836+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117206,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:44:01 2023","interfaces":[{"interface":"back","average":{"1min":0.27200000000000002,"5min":0.27000000000000002,"15min":0.26400000000000001},"min":{"1min":0.20399999999999999,"5min":0.17499999999999999,"15min":0.16},"max":{"1min":0.38400000000000001,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.216},{"interface":"front","average":{"1min":0.27600000000000002,"5min":0.27800000000000002,"15min":0.27100000000000002},"min":{"1min":0.23100000000000001,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.40400000000000003,"5min":0.47199999999999998,"15min":0.47199999999999998},"last":0.23999999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051796,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:43:55 2023","interfaces":[{"interface":"back","average":{"1min":0.26100000000000001,"5min":0.26700000000000002,"15min":0.27000000000000002},"min":{"1min":0.17799999999999999,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.374,"5min":0.44,"15min":0.44},"last":0.25600000000000001},{"interface":"front","average":{"1min":0.28299999999999997,"5min":0.28100000000000003,"15min":0.28399999999999997},"min":{"1min":0.20899999999999999,"5min":0.17599999999999999,"15min":0.16600000000000001},"max":{"1min":0.40100000000000002,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.28000000000000003}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:45:05.762 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:45:06.073 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all

Ignoring malparsed backtrace: 2023-03-25T04:45:25.658 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":624,"stamp":"2023-03-25T04:45:24.304294+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002366"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":41,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:42:34.382196+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:42:34.382196+0000","last_peered":"2023-03-25T04:42:34.382196+0000","last_clean":"2023-03-25T04:42:34.382196+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:42:34.382196+0000","last_undegraded":"2023-03-25T04:42:34.382196+0000","last_fullsized":"2023-03-25T04:42:34.382196+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:42:29.381744+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:42:29.381744+0000","last_peered":"2023-03-25T04:42:29.381744+0000","last_clean":"2023-03-25T04:42:29.381744+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:42:29.381744+0000","last_undegraded":"2023-03-25T04:42:29.381744+0000","last_fullsized":"2023-03-25T04:42:29.381744+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:23.375106+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:41:23.375106+0000","last_peered":"2023-03-25T04:41:23.375106+0000","last_clean":"2023-03-25T04:41:23.375106+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:41:23.375106+0000","last_undegraded":"2023-03-25T04:41:23.375106+0000","last_fullsized":"2023-03-25T04:41:23.375106+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:50.618930+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:41:50.618930+0000","last_peered":"2023-03-25T04:41:50.618930+0000","last_clean":"2023-03-25T04:41:50.618930+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:41:50.618930+0000","last_undegraded":"2023-03-25T04:41:50.618930+0000","last_fullsized":"2023-03-25T04:41:50.618930+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":41,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:58.377821+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:41:58.377821+0000","last_peered":"2023-03-25T04:41:58.377821+0000","last_clean":"2023-03-25T04:41:58.377821+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:41:58.377821+0000","last_undegraded":"2023-03-25T04:41:58.377821+0000","last_fullsized":"2023-03-25T04:41:58.377821+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:58.377836+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:41:58.377836+0000","last_peered":"2023-03-25T04:41:58.377836+0000","last_clean":"2023-03-25T04:41:58.377836+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:41:58.377836+0000","last_undegraded":"2023-03-25T04:41:58.377836+0000","last_fullsized":"2023-03-25T04:41:58.377836+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117213,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:45:01 2023","interfaces":[{"interface":"back","average":{"1min":0.251,"5min":0.26800000000000002,"15min":0.26400000000000001},"min":{"1min":0.183,"5min":0.17499999999999999,"15min":0.16},"max":{"1min":0.41499999999999998,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.26900000000000002},{"interface":"front","average":{"1min":0.28000000000000003,"5min":0.27900000000000003,"15min":0.27100000000000002},"min":{"1min":0.20999999999999999,"5min":0.185,"15min":0.17499999999999999},"max":{"1min":0.44400000000000001,"5min":0.47199999999999998,"15min":0.47199999999999998},"last":0.27900000000000003}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051803,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:44:55 2023","interfaces":[{"interface":"back","average":{"1min":0.252,"5min":0.26500000000000001,"15min":0.26900000000000002},"min":{"1min":0.183,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.39700000000000002,"5min":0.44,"15min":0.44},"last":0.20699999999999999},{"interface":"front","average":{"1min":0.27000000000000002,"5min":0.28100000000000003,"15min":0.28299999999999997},"min":{"1min":0.20300000000000001,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":0.42099999999999999,"5min":0.47599999999999998,"15min":0.48599999999999999},"last":0.23599999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:45:25.659 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:45:25.969 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:45:42.735+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:45:42.744+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:45:42.897 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace: 2023-03-25T04:46:00.183 INFO:teuthology.orchestra.run.smithi070.stdout:
, 2023-03-25T04:46:00.183 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":1217,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":20,"modified":"2023-03-25T04:45:42.307700+0000","services":{"mgr":{"daemons":{"summary":"","x":{"start_epoch":0,"start_stamp":"0.000000","gid":0,"addr":"(unrecognized address family 0)/0","metadata":{},"task_status":{}}}}}},"progress_events":{}}
, 2023-03-25T04:46:00.197 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now

Ignoring malparsed backtrace: 2023-03-25T04:46:19.910+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:46:20.039 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all

Ignoring malparsed backtrace: 2023-03-25T04:46:39.611 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}
, 2023-03-25T04:46:39.611 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json
, 2023-03-25T04:46:39.613+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:46:58.485+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:46:58.764+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:46:59.519+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:47:14.132 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:47:14.146 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":678,"stamp":"2023-03-25T04:47:12.325044+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002571"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":42,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:44.393426+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:44:44.393426+0000","last_peered":"2023-03-25T04:44:44.393426+0000","last_clean":"2023-03-25T04:44:44.393426+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:44:44.393426+0000","last_undegraded":"2023-03-25T04:44:44.393426+0000","last_fullsized":"2023-03-25T04:44:44.393426+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:39.393044+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:44:39.393044+0000","last_peered":"2023-03-25T04:44:39.393044+0000","last_clean":"2023-03-25T04:44:39.393044+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:44:39.393044+0000","last_undegraded":"2023-03-25T04:44:39.393044+0000","last_fullsized":"2023-03-25T04:44:39.393044+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:43:24.386177+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:43:24.386177+0000","last_peered":"2023-03-25T04:43:24.386177+0000","last_clean":"2023-03-25T04:43:24.386177+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:43:24.386177+0000","last_undegraded":"2023-03-25T04:43:24.386177+0000","last_fullsized":"2023-03-25T04:43:24.386177+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:43:55.629798+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:43:55.629798+0000","last_peered":"2023-03-25T04:43:55.629798+0000","last_clean":"2023-03-25T04:43:55.629798+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:43:55.629798+0000","last_undegraded":"2023-03-25T04:43:55.629798+0000","last_fullsized":"2023-03-25T04:43:55.629798+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":42,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:04.389904+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:44:04.389904+0000","last_peered":"2023-03-25T04:44:04.389904+0000","last_clean":"2023-03-25T04:44:04.389904+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:44:04.389904+0000","last_undegraded":"2023-03-25T04:44:04.389904+0000","last_fullsized":"2023-03-25T04:44:04.389904+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:04.389923+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:44:04.389923+0000","last_peered":"2023-03-25T04:44:04.389923+0000","last_clean":"2023-03-25T04:44:04.389923+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:44:04.389923+0000","last_undegraded":"2023-03-25T04:44:04.389923+0000","last_fullsized":"2023-03-25T04:44:04.389923+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117235,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:47:05 2023","interfaces":[{"interface":"back","average":{"1min":0.27800000000000002,"5min":0.27000000000000002,"15min":0.26300000000000001},"min":{"1min":0.20100000000000001,"5min":0.16900000000000001,"15min":0.16},"max":{"1min":0.39200000000000002,"5min":0.42999999999999999,"15min":0.45600000000000002},"last":0.379},{"interface":"front","average":{"1min":0.28399999999999997,"5min":0.28100000000000003,"15min":0.27200000000000002},"min":{"1min":0.17799999999999999,"5min":0.17799999999999999,"15min":0.17699999999999999},"max":{"1min":0.42599999999999999,"5min":0.47199999999999998,"15min":0.47199999999999998},"last":0.39800000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051825,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:47:01 2023","interfaces":[{"interface":"back","average":{"1min":0.27300000000000002,"5min":0.26100000000000001,"15min":0.27000000000000002},"min":{"1min":0.186,"5min":0.17399999999999999,"15min":0.17000000000000001},"max":{"1min":0.40400000000000003,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.23899999999999999},{"interface":"front","average":{"1min":0.28699999999999998,"5min":0.27500000000000002,"15min":0.28299999999999997},"min":{"1min":0.186,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.48599999999999999},"last":0.19600000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:47:31.198 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json
, 2023-03-25T04:47:31.411+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:47:31.789 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:47:49.509 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":696,"stamp":"2023-03-25T04:47:48.332093+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002697"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":42,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:44.393426+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:44:44.393426+0000","last_peered":"2023-03-25T04:44:44.393426+0000","last_clean":"2023-03-25T04:44:44.393426+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:44:44.393426+0000","last_undegraded":"2023-03-25T04:44:44.393426+0000","last_fullsized":"2023-03-25T04:44:44.393426+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:39.393044+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:44:39.393044+0000","last_peered":"2023-03-25T04:44:39.393044+0000","last_clean":"2023-03-25T04:44:39.393044+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:44:39.393044+0000","last_undegraded":"2023-03-25T04:44:39.393044+0000","last_fullsized":"2023-03-25T04:44:39.393044+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:45:24.396717+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:45:24.396717+0000","last_peered":"2023-03-25T04:45:24.396717+0000","last_clean":"2023-03-25T04:45:24.396717+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:45:24.396717+0000","last_undegraded":"2023-03-25T04:45:24.396717+0000","last_fullsized":"2023-03-25T04:45:24.396717+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:43:55.629798+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:43:55.629798+0000","last_peered":"2023-03-25T04:43:55.629798+0000","last_clean":"2023-03-25T04:43:55.629798+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:43:55.629798+0000","last_undegraded":"2023-03-25T04:43:55.629798+0000","last_fullsized":"2023-03-25T04:43:55.629798+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":42,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:04.389904+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:44:04.389904+0000","last_peered":"2023-03-25T04:44:04.389904+0000","last_clean":"2023-03-25T04:44:04.389904+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:44:04.389904+0000","last_undegraded":"2023-03-25T04:44:04.389904+0000","last_fullsized":"2023-03-25T04:44:04.389904+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:04.389923+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:44:04.389923+0000","last_peered":"2023-03-25T04:44:04.389923+0000","last_clean":"2023-03-25T04:44:04.389923+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:44:04.389923+0000","last_undegraded":"2023-03-25T04:44:04.389923+0000","last_fullsized":"2023-03-25T04:44:04.389923+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117242,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:47:05 2023","interfaces":[{"interface":"back","average":{"1min":0.27800000000000002,"5min":0.27000000000000002,"15min":0.26300000000000001},"min":{"1min":0.20100000000000001,"5min":0.16900000000000001,"15min":0.16},"max":{"1min":0.39200000000000002,"5min":0.42999999999999999,"15min":0.45600000000000002},"last":0.40500000000000003},{"interface":"front","average":{"1min":0.28399999999999997,"5min":0.28100000000000003,"15min":0.27200000000000002},"min":{"1min":0.17799999999999999,"5min":0.17799999999999999,"15min":0.17699999999999999},"max":{"1min":0.42599999999999999,"5min":0.47199999999999998,"15min":0.47199999999999998},"last":0.42399999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051834,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:47:01 2023","interfaces":[{"interface":"back","average":{"1min":0.27300000000000002,"5min":0.26100000000000001,"15min":0.27000000000000002},"min":{"1min":0.186,"5min":0.17399999999999999,"15min":0.17000000000000001},"max":{"1min":0.40400000000000003,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.25600000000000001},{"interface":"front","average":{"1min":0.28699999999999998,"5min":0.27500000000000002,"15min":0.28299999999999997},"min":{"1min":0.186,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.48599999999999999},"last":0.22800000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:47:49.510 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace:     ctx.managers[config['cluster']].wait_for_clean()
,   File "/home/teuthworker/src/github.com_ceph_ceph-c_511015c23eef0684a64171b710e432448df2b7c7/qa/tasks/ceph_manager.py", line 2747, in wait_for_clean
,     assert time.time() - start < timeout, \

Ignoring malparsed backtrace:  ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
,  1: /lib64/libpthread.so.0(+0x12cf0) [0x7f1453c2fcf0]
,  2: __pthread_mutex_lock()

Ignoring malparsed backtrace: 2023-03-25T04:28:05.172 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":142,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":5,"modified":"2023-03-25T04:27:12.092098+0000","services":{}},"progress_events":{}}
, 2023-03-25T04:28:05.185 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now
, 2023-03-25T04:28:05.186 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace: 2023-03-25T04:28:28.360 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":116,"stamp":"2023-03-25T04:28:28.108007+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002578"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.314661+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:27:45.314661+0000","last_peered":"2023-03-25T04:27:45.314661+0000","last_clean":"2023-03-25T04:27:45.314661+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:27:45.314661+0000","last_undegraded":"2023-03-25T04:27:45.314661+0000","last_fullsized":"2023-03-25T04:27:45.314661+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":25,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.310745+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:27:44.310745+0000","last_peered":"2023-03-25T04:27:44.310745+0000","last_clean":"2023-03-25T04:27:44.310745+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:27:44.310745+0000","last_undegraded":"2023-03-25T04:27:44.310745+0000","last_fullsized":"2023-03-25T04:27:44.310745+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":25,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.310650+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:27:44.310650+0000","last_peered":"2023-03-25T04:27:44.310650+0000","last_clean":"2023-03-25T04:27:44.310650+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:27:44.310650+0000","last_undegraded":"2023-03-25T04:27:44.310650+0000","last_fullsized":"2023-03-25T04:27:44.310650+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":26,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.312962+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:27:44.312962+0000","last_peered":"2023-03-25T04:27:44.312962+0000","last_clean":"2023-03-25T04:27:44.312962+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:27:44.312962+0000","last_undegraded":"2023-03-25T04:27:44.312962+0000","last_fullsized":"2023-03-25T04:27:44.312962+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315064+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:27:45.315064+0000","last_peered":"2023-03-25T04:27:45.315064+0000","last_clean":"2023-03-25T04:27:45.315064+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:27:45.315064+0000","last_undegraded":"2023-03-25T04:27:45.315064+0000","last_fullsized":"2023-03-25T04:27:45.315064+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":25,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.310707+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:27:44.310707+0000","last_peered":"2023-03-25T04:27:44.310707+0000","last_clean":"2023-03-25T04:27:44.310707+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:27:44.310707+0000","last_undegraded":"2023-03-25T04:27:44.310707+0000","last_fullsized":"2023-03-25T04:27:44.310707+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117007,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.375}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051597,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.186}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:28:28.362 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean
, 2023-03-25T04:28:28.362 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:28:45.655 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":124,"stamp":"2023-03-25T04:28:44.111265+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002492"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.314661+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:27:45.314661+0000","last_peered":"2023-03-25T04:27:45.314661+0000","last_clean":"2023-03-25T04:27:45.314661+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:27:45.314661+0000","last_undegraded":"2023-03-25T04:27:45.314661+0000","last_fullsized":"2023-03-25T04:27:45.314661+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":25,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.310745+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:27:44.310745+0000","last_peered":"2023-03-25T04:27:44.310745+0000","last_clean":"2023-03-25T04:27:44.310745+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:27:44.310745+0000","last_undegraded":"2023-03-25T04:27:44.310745+0000","last_fullsized":"2023-03-25T04:27:44.310745+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":25,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.310650+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:27:44.310650+0000","last_peered":"2023-03-25T04:27:44.310650+0000","last_clean":"2023-03-25T04:27:44.310650+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:27:44.310650+0000","last_undegraded":"2023-03-25T04:27:44.310650+0000","last_fullsized":"2023-03-25T04:27:44.310650+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":26,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.312962+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:27:44.312962+0000","last_peered":"2023-03-25T04:27:44.312962+0000","last_clean":"2023-03-25T04:27:44.312962+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:27:44.312962+0000","last_undegraded":"2023-03-25T04:27:44.312962+0000","last_fullsized":"2023-03-25T04:27:44.312962+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315064+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:27:45.315064+0000","last_peered":"2023-03-25T04:27:45.315064+0000","last_clean":"2023-03-25T04:27:45.315064+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:27:45.315064+0000","last_undegraded":"2023-03-25T04:27:45.315064+0000","last_fullsized":"2023-03-25T04:27:45.315064+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":25,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.310707+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:27:44.310707+0000","last_peered":"2023-03-25T04:27:44.310707+0000","last_clean":"2023-03-25T04:27:44.310707+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:27:44.310707+0000","last_undegraded":"2023-03-25T04:27:44.310707+0000","last_fullsized":"2023-03-25T04:27:44.310707+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117011,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:28:31 2023","interfaces":[{"interface":"back","average":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.36899999999999999},{"interface":"front","average":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"min":{"1min":0.19,"5min":0.19,"15min":0.19},"max":{"1min":0.46300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.39800000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051601,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:28:24 2023","interfaces":[{"interface":"back","average":{"1min":0.249,"5min":0.249,"15min":0.249},"min":{"1min":0.17699999999999999,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"last":0.254},{"interface":"front","average":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"min":{"1min":0.20200000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"last":0.312}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:28:45.656 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace: 2023-03-25T04:29:05.171 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}
, 2023-03-25T04:29:05.171 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json
, 2023-03-25T04:29:05.561 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:29:22.465 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:29:22.773 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all

Ignoring malparsed backtrace: 2023-03-25T04:29:42.405 INFO:teuthology.orchestra.run.smithi070.stdout:
, 2023-03-25T04:29:42.406 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":240,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":9,"modified":"2023-03-25T04:29:32.120358+0000","services":{"mon":{"daemons":{"summary":"","a":{"start_epoch":0,"start_stamp":"0.000000","gid":0,"addr":"(unrecognized address family 0)/0","metadata":{},"task_status":{}}}}}},"progress_events":{}}
, 2023-03-25T04:29:42.419 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:29:59.627 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":161,"stamp":"2023-03-25T04:29:58.125186+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002398"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315468+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:27:45.315468+0000","last_peered":"2023-03-25T04:27:45.315468+0000","last_clean":"2023-03-25T04:27:45.315468+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:27:45.315468+0000","last_undegraded":"2023-03-25T04:27:45.315468+0000","last_fullsized":"2023-03-25T04:27:45.315468+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312845+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:27:45.312845+0000","last_peered":"2023-03-25T04:27:45.312845+0000","last_clean":"2023-03-25T04:27:45.312845+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:27:45.312845+0000","last_undegraded":"2023-03-25T04:27:45.312845+0000","last_fullsized":"2023-03-25T04:27:45.312845+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312882+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:27:45.312882+0000","last_peered":"2023-03-25T04:27:45.312882+0000","last_clean":"2023-03-25T04:27:45.312882+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:27:45.312882+0000","last_undegraded":"2023-03-25T04:27:45.312882+0000","last_fullsized":"2023-03-25T04:27:45.312882+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.311730+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:27:45.311730+0000","last_peered":"2023-03-25T04:27:45.311730+0000","last_clean":"2023-03-25T04:27:45.311730+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:27:45.311730+0000","last_undegraded":"2023-03-25T04:27:45.311730+0000","last_fullsized":"2023-03-25T04:27:45.311730+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315735+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:27:45.315735+0000","last_peered":"2023-03-25T04:27:45.315735+0000","last_clean":"2023-03-25T04:27:45.315735+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:27:45.315735+0000","last_undegraded":"2023-03-25T04:27:45.315735+0000","last_fullsized":"2023-03-25T04:27:45.315735+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312833+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:27:45.312833+0000","last_peered":"2023-03-25T04:27:45.312833+0000","last_clean":"2023-03-25T04:27:45.312833+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:27:45.312833+0000","last_undegraded":"2023-03-25T04:27:45.312833+0000","last_fullsized":"2023-03-25T04:27:45.312833+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117026,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:29:33 2023","interfaces":[{"interface":"back","average":{"1min":0.26300000000000001,"5min":0.29099999999999998,"15min":0.29599999999999999},"min":{"1min":0.191,"5min":0.191,"15min":0.191},"max":{"1min":0.371,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.20499999999999999},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.29599999999999999,"15min":0.29999999999999999},"min":{"1min":0.21099999999999999,"5min":0.19,"15min":0.19},"max":{"1min":0.39800000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.19500000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051616,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:29:29 2023","interfaces":[{"interface":"back","average":{"1min":0.27200000000000002,"5min":0.253,"15min":0.25},"min":{"1min":0.183,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"last":0.26400000000000001},{"interface":"front","average":{"1min":0.28499999999999998,"5min":0.26800000000000002,"15min":0.26500000000000001},"min":{"1min":0.16600000000000001,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"last":0.29899999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:29:59.629 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean

Ignoring malparsed backtrace: 2023-03-25T04:30:16.889 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:30:16.903 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":170,"stamp":"2023-03-25T04:30:16.128427+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002222"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315468+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:27:45.315468+0000","last_peered":"2023-03-25T04:27:45.315468+0000","last_clean":"2023-03-25T04:27:45.315468+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:27:45.315468+0000","last_undegraded":"2023-03-25T04:27:45.315468+0000","last_fullsized":"2023-03-25T04:27:45.315468+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312845+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:27:45.312845+0000","last_peered":"2023-03-25T04:27:45.312845+0000","last_clean":"2023-03-25T04:27:45.312845+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:27:45.312845+0000","last_undegraded":"2023-03-25T04:27:45.312845+0000","last_fullsized":"2023-03-25T04:27:45.312845+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312882+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:27:45.312882+0000","last_peered":"2023-03-25T04:27:45.312882+0000","last_clean":"2023-03-25T04:27:45.312882+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:27:45.312882+0000","last_undegraded":"2023-03-25T04:27:45.312882+0000","last_fullsized":"2023-03-25T04:27:45.312882+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.311730+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:27:45.311730+0000","last_peered":"2023-03-25T04:27:45.311730+0000","last_clean":"2023-03-25T04:27:45.311730+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:27:45.311730+0000","last_undegraded":"2023-03-25T04:27:45.311730+0000","last_fullsized":"2023-03-25T04:27:45.311730+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315735+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:27:45.315735+0000","last_peered":"2023-03-25T04:27:45.315735+0000","last_clean":"2023-03-25T04:27:45.315735+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:27:45.315735+0000","last_undegraded":"2023-03-25T04:27:45.315735+0000","last_fullsized":"2023-03-25T04:27:45.315735+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312833+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:27:45.312833+0000","last_peered":"2023-03-25T04:27:45.312833+0000","last_clean":"2023-03-25T04:27:45.312833+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:27:45.312833+0000","last_undegraded":"2023-03-25T04:27:45.312833+0000","last_fullsized":"2023-03-25T04:27:45.312833+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117030,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:29:33 2023","interfaces":[{"interface":"back","average":{"1min":0.26300000000000001,"5min":0.29099999999999998,"15min":0.29599999999999999},"min":{"1min":0.191,"5min":0.191,"15min":0.191},"max":{"1min":0.371,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.245},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.29599999999999999,"15min":0.29999999999999999},"min":{"1min":0.21099999999999999,"5min":0.19,"15min":0.19},"max":{"1min":0.39800000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.20899999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051620,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:29:29 2023","interfaces":[{"interface":"back","average":{"1min":0.27200000000000002,"5min":0.253,"15min":0.25},"min":{"1min":0.183,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"last":0.19400000000000001},{"interface":"front","average":{"1min":0.28499999999999998,"5min":0.26800000000000002,"15min":0.26500000000000001},"min":{"1min":0.16600000000000001,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"last":0.23599999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:30:31.984+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:30:32.201+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:30:32.981+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)

Ignoring malparsed backtrace: 2023-03-25T04:30:48.449 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now
, 2023-03-25T04:30:48.450 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:30:48.760 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:31:08.342 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":196,"stamp":"2023-03-25T04:31:08.139018+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002431"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315468+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:27:45.315468+0000","last_peered":"2023-03-25T04:27:45.315468+0000","last_clean":"2023-03-25T04:27:45.315468+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:27:45.315468+0000","last_undegraded":"2023-03-25T04:27:45.315468+0000","last_fullsized":"2023-03-25T04:27:45.315468+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312845+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:27:45.312845+0000","last_peered":"2023-03-25T04:27:45.312845+0000","last_clean":"2023-03-25T04:27:45.312845+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:27:45.312845+0000","last_undegraded":"2023-03-25T04:27:45.312845+0000","last_fullsized":"2023-03-25T04:27:45.312845+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312882+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:27:45.312882+0000","last_peered":"2023-03-25T04:27:45.312882+0000","last_clean":"2023-03-25T04:27:45.312882+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:27:45.312882+0000","last_undegraded":"2023-03-25T04:27:45.312882+0000","last_fullsized":"2023-03-25T04:27:45.312882+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.311730+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:27:45.311730+0000","last_peered":"2023-03-25T04:27:45.311730+0000","last_clean":"2023-03-25T04:27:45.311730+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:27:45.311730+0000","last_undegraded":"2023-03-25T04:27:45.311730+0000","last_fullsized":"2023-03-25T04:27:45.311730+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315735+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:27:45.315735+0000","last_peered":"2023-03-25T04:27:45.315735+0000","last_clean":"2023-03-25T04:27:45.315735+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:27:45.315735+0000","last_undegraded":"2023-03-25T04:27:45.315735+0000","last_fullsized":"2023-03-25T04:27:45.315735+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312833+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:27:45.312833+0000","last_peered":"2023-03-25T04:27:45.312833+0000","last_clean":"2023-03-25T04:27:45.312833+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:27:45.312833+0000","last_undegraded":"2023-03-25T04:27:45.312833+0000","last_fullsized":"2023-03-25T04:27:45.312833+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117040,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:30:37 2023","interfaces":[{"interface":"back","average":{"1min":0.28799999999999998,"5min":0.28899999999999998,"15min":0.29499999999999998},"min":{"1min":0.20200000000000001,"5min":0.191,"15min":0.191},"max":{"1min":0.433,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.29799999999999999},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.28999999999999998,"15min":0.29799999999999999},"min":{"1min":0.17499999999999999,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":0.40300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.25700000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051630,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:30:32 2023","interfaces":[{"interface":"back","average":{"1min":0.25600000000000001,"5min":0.255,"15min":0.251},"min":{"1min":0.19400000000000001,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.35699999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"last":0.22800000000000001},{"interface":"front","average":{"1min":0.28699999999999998,"5min":0.27200000000000002,"15min":0.26600000000000001},"min":{"1min":0.20599999999999999,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.39400000000000002,"5min":0.42899999999999999,"15min":0.42899999999999999},"last":0.24099999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:31:08.343 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace: 2023-03-25T04:31:24.283+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:31:25.192+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:31:25.295+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)

Ignoring malparsed backtrace: 2023-03-25T04:31:40.257 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:31:40.271 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":212,"stamp":"2023-03-25T04:31:40.144732+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002208"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315468+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:27:45.315468+0000","last_peered":"2023-03-25T04:27:45.315468+0000","last_clean":"2023-03-25T04:27:45.315468+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:27:45.315468+0000","last_undegraded":"2023-03-25T04:27:45.315468+0000","last_fullsized":"2023-03-25T04:27:45.315468+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312845+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:27:45.312845+0000","last_peered":"2023-03-25T04:27:45.312845+0000","last_clean":"2023-03-25T04:27:45.312845+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:27:45.312845+0000","last_undegraded":"2023-03-25T04:27:45.312845+0000","last_fullsized":"2023-03-25T04:27:45.312845+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:23.308607+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:29:23.308607+0000","last_peered":"2023-03-25T04:29:23.308607+0000","last_clean":"2023-03-25T04:29:23.308607+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:29:23.308607+0000","last_undegraded":"2023-03-25T04:29:23.308607+0000","last_fullsized":"2023-03-25T04:29:23.308607+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":28,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:20.554710+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:29:20.554710+0000","last_peered":"2023-03-25T04:29:20.554710+0000","last_clean":"2023-03-25T04:29:20.554710+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:29:20.554710+0000","last_undegraded":"2023-03-25T04:29:20.554710+0000","last_fullsized":"2023-03-25T04:29:20.554710+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309216+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:29:28.309216+0000","last_peered":"2023-03-25T04:29:28.309216+0000","last_clean":"2023-03-25T04:29:28.309216+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:29:28.309216+0000","last_undegraded":"2023-03-25T04:29:28.309216+0000","last_fullsized":"2023-03-25T04:29:28.309216+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309233+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:29:28.309233+0000","last_peered":"2023-03-25T04:29:28.309233+0000","last_clean":"2023-03-25T04:29:28.309233+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:29:28.309233+0000","last_undegraded":"2023-03-25T04:29:28.309233+0000","last_fullsized":"2023-03-25T04:29:28.309233+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117046,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:30:37 2023","interfaces":[{"interface":"back","average":{"1min":0.28799999999999998,"5min":0.28899999999999998,"15min":0.29499999999999998},"min":{"1min":0.20200000000000001,"5min":0.191,"15min":0.191},"max":{"1min":0.433,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.216},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.28999999999999998,"15min":0.29799999999999999},"min":{"1min":0.17499999999999999,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":0.40300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.24299999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051637,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:31:35 2023","interfaces":[{"interface":"back","average":{"1min":0.26100000000000001,"5min":0.25700000000000001,"15min":0.251},"min":{"1min":0.19,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.35899999999999999,"5min":0.41599999999999998,"15min":0.41599999999999998},"last":0.221},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.27400000000000002,"15min":0.26700000000000002},"min":{"1min":0.20200000000000001,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.374,"5min":0.42899999999999999,"15min":0.42899999999999999},"last":0.24099999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:32:01.906+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:32:02.388+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:32:02.794 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:32:22.326 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":233,"stamp":"2023-03-25T04:32:22.153289+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002545"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:33.309748+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:29:33.309748+0000","last_peered":"2023-03-25T04:29:33.309748+0000","last_clean":"2023-03-25T04:29:33.309748+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:29:33.309748+0000","last_undegraded":"2023-03-25T04:29:33.309748+0000","last_fullsized":"2023-03-25T04:29:33.309748+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:33.309729+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:29:33.309729+0000","last_peered":"2023-03-25T04:29:33.309729+0000","last_clean":"2023-03-25T04:29:33.309729+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:29:33.309729+0000","last_undegraded":"2023-03-25T04:29:33.309729+0000","last_fullsized":"2023-03-25T04:29:33.309729+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:23.308607+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:29:23.308607+0000","last_peered":"2023-03-25T04:29:23.308607+0000","last_clean":"2023-03-25T04:29:23.308607+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:29:23.308607+0000","last_undegraded":"2023-03-25T04:29:23.308607+0000","last_fullsized":"2023-03-25T04:29:23.308607+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":28,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:20.554710+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:29:20.554710+0000","last_peered":"2023-03-25T04:29:20.554710+0000","last_clean":"2023-03-25T04:29:20.554710+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:29:20.554710+0000","last_undegraded":"2023-03-25T04:29:20.554710+0000","last_fullsized":"2023-03-25T04:29:20.554710+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309216+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:29:28.309216+0000","last_peered":"2023-03-25T04:29:28.309216+0000","last_clean":"2023-03-25T04:29:28.309216+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:29:28.309216+0000","last_undegraded":"2023-03-25T04:29:28.309216+0000","last_fullsized":"2023-03-25T04:29:28.309216+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309233+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:29:28.309233+0000","last_peered":"2023-03-25T04:29:28.309233+0000","last_clean":"2023-03-25T04:29:28.309233+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:29:28.309233+0000","last_undegraded":"2023-03-25T04:29:28.309233+0000","last_fullsized":"2023-03-25T04:29:28.309233+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117055,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:31:42 2023","interfaces":[{"interface":"back","average":{"1min":0.27500000000000002,"5min":0.28399999999999997,"15min":0.29399999999999998},"min":{"1min":0.19800000000000001,"5min":0.191,"15min":0.191},"max":{"1min":0.36899999999999999,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.32300000000000001},{"interface":"front","average":{"1min":0.27900000000000003,"5min":0.28499999999999998,"15min":0.29599999999999999},"min":{"1min":0.216,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":0.40799999999999997,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.27100000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051646,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:31:35 2023","interfaces":[{"interface":"back","average":{"1min":0.26100000000000001,"5min":0.25700000000000001,"15min":0.251},"min":{"1min":0.19,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.35899999999999999,"5min":0.41599999999999998,"15min":0.41599999999999998},"last":0.25600000000000001},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.27400000000000002,"15min":0.26700000000000002},"min":{"1min":0.20200000000000001,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.374,"5min":0.42899999999999999,"15min":0.42899999999999999},"last":0.29199999999999998}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:32:22.327 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean
, 2023-03-25T04:32:22.327 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}

Ignoring malparsed backtrace: 2023-03-25T04:32:39.836 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:32:39.849 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":241,"stamp":"2023-03-25T04:32:38.156312+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002346"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:33.309748+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:29:33.309748+0000","last_peered":"2023-03-25T04:29:33.309748+0000","last_clean":"2023-03-25T04:29:33.309748+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:29:33.309748+0000","last_undegraded":"2023-03-25T04:29:33.309748+0000","last_fullsized":"2023-03-25T04:29:33.309748+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:33.309729+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:29:33.309729+0000","last_peered":"2023-03-25T04:29:33.309729+0000","last_clean":"2023-03-25T04:29:33.309729+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:29:33.309729+0000","last_undegraded":"2023-03-25T04:29:33.309729+0000","last_fullsized":"2023-03-25T04:29:33.309729+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:23.308607+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:29:23.308607+0000","last_peered":"2023-03-25T04:29:23.308607+0000","last_clean":"2023-03-25T04:29:23.308607+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:29:23.308607+0000","last_undegraded":"2023-03-25T04:29:23.308607+0000","last_fullsized":"2023-03-25T04:29:23.308607+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":28,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:20.554710+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:29:20.554710+0000","last_peered":"2023-03-25T04:29:20.554710+0000","last_clean":"2023-03-25T04:29:20.554710+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:29:20.554710+0000","last_undegraded":"2023-03-25T04:29:20.554710+0000","last_fullsized":"2023-03-25T04:29:20.554710+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309216+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:29:28.309216+0000","last_peered":"2023-03-25T04:29:28.309216+0000","last_clean":"2023-03-25T04:29:28.309216+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:29:28.309216+0000","last_undegraded":"2023-03-25T04:29:28.309216+0000","last_fullsized":"2023-03-25T04:29:28.309216+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309233+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:29:28.309233+0000","last_peered":"2023-03-25T04:29:28.309233+0000","last_clean":"2023-03-25T04:29:28.309233+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:29:28.309233+0000","last_undegraded":"2023-03-25T04:29:28.309233+0000","last_fullsized":"2023-03-25T04:29:28.309233+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117058,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:31:42 2023","interfaces":[{"interface":"back","average":{"1min":0.27500000000000002,"5min":0.28399999999999997,"15min":0.29399999999999998},"min":{"1min":0.19800000000000001,"5min":0.191,"15min":0.191},"max":{"1min":0.36899999999999999,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.30599999999999999},{"interface":"front","average":{"1min":0.27900000000000003,"5min":0.28499999999999998,"15min":0.29599999999999999},"min":{"1min":0.216,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":0.40799999999999997,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.32500000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051649,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:31:35 2023","interfaces":[{"interface":"back","average":{"1min":0.26100000000000001,"5min":0.25700000000000001,"15min":0.251},"min":{"1min":0.19,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.35899999999999999,"5min":0.41599999999999998,"15min":0.41599999999999998},"last":0.24199999999999999},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.27400000000000002,"15min":0.26700000000000002},"min":{"1min":0.20200000000000001,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.374,"5min":0.42899999999999999,"15min":0.42899999999999999},"last":0.22600000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:32:54.944+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:32:55.469+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:32:55.927+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)

Ignoring malparsed backtrace: 2023-03-25T04:33:13.958 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean
, 2023-03-25T04:33:13.959 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}
, 2023-03-25T04:33:13.959 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json

Ignoring malparsed backtrace: 2023-03-25T04:33:31.199 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":267,"stamp":"2023-03-25T04:33:30.166660+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002269"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:33.309748+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:29:33.309748+0000","last_peered":"2023-03-25T04:29:33.309748+0000","last_clean":"2023-03-25T04:29:33.309748+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:29:33.309748+0000","last_undegraded":"2023-03-25T04:29:33.309748+0000","last_fullsized":"2023-03-25T04:29:33.309748+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:33.309729+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:29:33.309729+0000","last_peered":"2023-03-25T04:29:33.309729+0000","last_clean":"2023-03-25T04:29:33.309729+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:29:33.309729+0000","last_undegraded":"2023-03-25T04:29:33.309729+0000","last_fullsized":"2023-03-25T04:29:33.309729+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":28,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:31:23.320894+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:31:23.320894+0000","last_peered":"2023-03-25T04:31:23.320894+0000","last_clean":"2023-03-25T04:31:23.320894+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:31:23.320894+0000","last_undegraded":"2023-03-25T04:31:23.320894+0000","last_fullsized":"2023-03-25T04:31:23.320894+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":28,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:20.554710+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:29:20.554710+0000","last_peered":"2023-03-25T04:29:20.554710+0000","last_clean":"2023-03-25T04:29:20.554710+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:29:20.554710+0000","last_undegraded":"2023-03-25T04:29:20.554710+0000","last_fullsized":"2023-03-25T04:29:20.554710+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309216+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:29:28.309216+0000","last_peered":"2023-03-25T04:29:28.309216+0000","last_clean":"2023-03-25T04:29:28.309216+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:29:28.309216+0000","last_undegraded":"2023-03-25T04:29:28.309216+0000","last_fullsized":"2023-03-25T04:29:28.309216+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309233+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:29:28.309233+0000","last_peered":"2023-03-25T04:29:28.309233+0000","last_clean":"2023-03-25T04:29:28.309233+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:29:28.309233+0000","last_undegraded":"2023-03-25T04:29:28.309233+0000","last_fullsized":"2023-03-25T04:29:28.309233+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117068,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:32:46 2023","interfaces":[{"interface":"back","average":{"1min":0.27000000000000002,"5min":0.27900000000000003,"15min":0.29199999999999998},"min":{"1min":0.17899999999999999,"5min":0.17899999999999999,"15min":0.17899999999999999},"max":{"1min":0.35599999999999998,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.28799999999999998},{"interface":"front","average":{"1min":0.26800000000000002,"5min":0.27800000000000002,"15min":0.29399999999999998},"min":{"1min":0.19900000000000001,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":0.377,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.315}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051660,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:32:36 2023","interfaces":[{"interface":"back","average":{"1min":0.248,"5min":0.25700000000000001,"15min":0.251},"min":{"1min":0.17000000000000001,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"last":0.22700000000000001},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.27600000000000002,"15min":0.26800000000000002},"min":{"1min":0.192,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"last":0.20399999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:33:31.201 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:33:31.400+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)

Ignoring malparsed backtrace: 2023-03-25T04:33:47.531+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:33:47.951+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:33:48.505+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)

Ignoring malparsed backtrace: 2023-03-25T04:34:06.104 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json
, 2023-03-25T04:34:06.118+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:34:06.497 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:34:23.399 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:34:23.665+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:34:23.712 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:34:40.195+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:34:40.523+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:34:40.676 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace: 2023-03-25T04:35:03.381+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:35:04.139+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:35:04.391+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)

Ignoring malparsed backtrace: 2023-03-25T04:35:22.738 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean
, 2023-03-25T04:35:22.739 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}
, 2023-03-25T04:35:22.739 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json

Ignoring malparsed backtrace: 2023-03-25T04:35:41.393+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:35:42.352+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:35:42.376+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:35:57.384 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":340,"stamp":"2023-03-25T04:35:56.194992+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.001930"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":36,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:31:43.322894+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:31:43.322894+0000","last_peered":"2023-03-25T04:31:43.322894+0000","last_clean":"2023-03-25T04:31:43.322894+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:31:43.322894+0000","last_undegraded":"2023-03-25T04:31:43.322894+0000","last_fullsized":"2023-03-25T04:31:43.322894+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":28,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:31:43.322876+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:31:43.322876+0000","last_peered":"2023-03-25T04:31:43.322876+0000","last_clean":"2023-03-25T04:31:43.322876+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:31:43.322876+0000","last_undegraded":"2023-03-25T04:31:43.322876+0000","last_fullsized":"2023-03-25T04:31:43.322876+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:23.331998+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:33:23.331998+0000","last_peered":"2023-03-25T04:33:23.331998+0000","last_clean":"2023-03-25T04:33:23.331998+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:33:23.331998+0000","last_undegraded":"2023-03-25T04:33:23.331998+0000","last_fullsized":"2023-03-25T04:33:23.331998+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:30.576723+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:33:30.576723+0000","last_peered":"2023-03-25T04:33:30.576723+0000","last_clean":"2023-03-25T04:33:30.576723+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:33:30.576723+0000","last_undegraded":"2023-03-25T04:33:30.576723+0000","last_fullsized":"2023-03-25T04:33:30.576723+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":37,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:38.333447+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:33:38.333447+0000","last_peered":"2023-03-25T04:33:38.333447+0000","last_clean":"2023-03-25T04:33:38.333447+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:33:38.333447+0000","last_undegraded":"2023-03-25T04:33:38.333447+0000","last_fullsized":"2023-03-25T04:33:38.333447+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:38.333467+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:33:38.333467+0000","last_peered":"2023-03-25T04:33:38.333467+0000","last_clean":"2023-03-25T04:33:38.333467+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:33:38.333467+0000","last_undegraded":"2023-03-25T04:33:38.333467+0000","last_fullsized":"2023-03-25T04:33:38.333467+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117098,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:35:49 2023","interfaces":[{"interface":"back","average":{"1min":0.255,"5min":0.26000000000000001,"15min":0.28199999999999997},"min":{"1min":0.17899999999999999,"5min":0.16,"15min":0.16},"max":{"1min":0.36399999999999999,"5min":0.42599999999999999,"15min":0.45100000000000001},"last":0.26100000000000001},{"interface":"front","average":{"1min":0.26000000000000001,"5min":0.26700000000000002,"15min":0.28599999999999998},"min":{"1min":0.188,"5min":0.17999999999999999,"15min":0.17499999999999999},"max":{"1min":0.39100000000000001,"5min":0.45200000000000001,"15min":0.46300000000000002},"last":0.28399999999999997}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051689,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:35:42 2023","interfaces":[{"interface":"back","average":{"1min":0.29399999999999998,"5min":0.27200000000000002,"15min":0.25800000000000001},"min":{"1min":0.17299999999999999,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.41099999999999998,"5min":0.42699999999999999,"15min":0.42699999999999999},"last":0.20399999999999999},{"interface":"front","average":{"1min":0.30299999999999999,"5min":0.28399999999999997,"15min":0.27300000000000002},"min":{"1min":0.19700000000000001,"5min":0.192,"15min":0.16600000000000001},"max":{"1min":0.43099999999999999,"5min":0.45800000000000002,"15min":0.45800000000000002},"last":0.22}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:35:57.624+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)

Ignoring malparsed backtrace: 2023-03-25T04:36:17.226 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:36:17.239 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":350,"stamp":"2023-03-25T04:36:16.198866+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002148"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":37,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:53.335026+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:33:53.335026+0000","last_peered":"2023-03-25T04:33:53.335026+0000","last_clean":"2023-03-25T04:33:53.335026+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:33:53.335026+0000","last_undegraded":"2023-03-25T04:33:53.335026+0000","last_fullsized":"2023-03-25T04:33:53.335026+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:53.335009+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:33:53.335009+0000","last_peered":"2023-03-25T04:33:53.335009+0000","last_clean":"2023-03-25T04:33:53.335009+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:33:53.335009+0000","last_undegraded":"2023-03-25T04:33:53.335009+0000","last_fullsized":"2023-03-25T04:33:53.335009+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:23.331998+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:33:23.331998+0000","last_peered":"2023-03-25T04:33:23.331998+0000","last_clean":"2023-03-25T04:33:23.331998+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:33:23.331998+0000","last_undegraded":"2023-03-25T04:33:23.331998+0000","last_fullsized":"2023-03-25T04:33:23.331998+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:30.576723+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:33:30.576723+0000","last_peered":"2023-03-25T04:33:30.576723+0000","last_clean":"2023-03-25T04:33:30.576723+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:33:30.576723+0000","last_undegraded":"2023-03-25T04:33:30.576723+0000","last_fullsized":"2023-03-25T04:33:30.576723+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":37,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:38.333447+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:33:38.333447+0000","last_peered":"2023-03-25T04:33:38.333447+0000","last_clean":"2023-03-25T04:33:38.333447+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:33:38.333447+0000","last_undegraded":"2023-03-25T04:33:38.333447+0000","last_fullsized":"2023-03-25T04:33:38.333447+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:38.333467+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:33:38.333467+0000","last_peered":"2023-03-25T04:33:38.333467+0000","last_clean":"2023-03-25T04:33:38.333467+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:33:38.333467+0000","last_undegraded":"2023-03-25T04:33:38.333467+0000","last_fullsized":"2023-03-25T04:33:38.333467+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117102,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:35:49 2023","interfaces":[{"interface":"back","average":{"1min":0.255,"5min":0.26000000000000001,"15min":0.28199999999999997},"min":{"1min":0.17899999999999999,"5min":0.16,"15min":0.16},"max":{"1min":0.36399999999999999,"5min":0.42599999999999999,"15min":0.45100000000000001},"last":0.28399999999999997},{"interface":"front","average":{"1min":0.26000000000000001,"5min":0.26700000000000002,"15min":0.28599999999999998},"min":{"1min":0.188,"5min":0.17999999999999999,"15min":0.17499999999999999},"max":{"1min":0.39100000000000001,"5min":0.45200000000000001,"15min":0.46300000000000002},"last":0.30299999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051693,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:35:42 2023","interfaces":[{"interface":"back","average":{"1min":0.29399999999999998,"5min":0.27200000000000002,"15min":0.25800000000000001},"min":{"1min":0.17299999999999999,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.41099999999999998,"5min":0.42699999999999999,"15min":0.42699999999999999},"last":0.35099999999999998},{"interface":"front","average":{"1min":0.30299999999999999,"5min":0.28399999999999997,"15min":0.27300000000000002},"min":{"1min":0.19700000000000001,"5min":0.192,"15min":0.16600000000000001},"max":{"1min":0.43099999999999999,"5min":0.45800000000000002,"15min":0.45800000000000002},"last":0.32900000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:36:34.556+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:36:34.717+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:36:35.583+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)

Ignoring malparsed backtrace: 2023-03-25T04:36:52.680 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json
, 2023-03-25T04:36:52.701+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:36:53.067 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:37:09.963 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:37:10.273 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all

Ignoring malparsed backtrace: 2023-03-25T04:37:26.856+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:37:27.288 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:37:27.538+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:37:41.903 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":392,"stamp":"2023-03-25T04:37:40.213373+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002012"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":37,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:53.335026+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:33:53.335026+0000","last_peered":"2023-03-25T04:33:53.335026+0000","last_clean":"2023-03-25T04:33:53.335026+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:33:53.335026+0000","last_undegraded":"2023-03-25T04:33:53.335026+0000","last_fullsized":"2023-03-25T04:33:53.335026+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:53.335009+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:33:53.335009+0000","last_peered":"2023-03-25T04:33:53.335009+0000","last_clean":"2023-03-25T04:33:53.335009+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:33:53.335009+0000","last_undegraded":"2023-03-25T04:33:53.335009+0000","last_fullsized":"2023-03-25T04:33:53.335009+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:23.343585+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:35:23.343585+0000","last_peered":"2023-03-25T04:35:23.343585+0000","last_clean":"2023-03-25T04:35:23.343585+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:35:23.343585+0000","last_undegraded":"2023-03-25T04:35:23.343585+0000","last_fullsized":"2023-03-25T04:35:23.343585+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:30.576723+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:33:30.576723+0000","last_peered":"2023-03-25T04:33:30.576723+0000","last_clean":"2023-03-25T04:33:30.576723+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:33:30.576723+0000","last_undegraded":"2023-03-25T04:33:30.576723+0000","last_fullsized":"2023-03-25T04:33:30.576723+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":37,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:38.333447+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:33:38.333447+0000","last_peered":"2023-03-25T04:33:38.333447+0000","last_clean":"2023-03-25T04:33:38.333447+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:33:38.333447+0000","last_undegraded":"2023-03-25T04:33:38.333447+0000","last_fullsized":"2023-03-25T04:33:38.333447+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:38.333467+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:33:38.333467+0000","last_peered":"2023-03-25T04:33:38.333467+0000","last_clean":"2023-03-25T04:33:38.333467+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:33:38.333467+0000","last_undegraded":"2023-03-25T04:33:38.333467+0000","last_fullsized":"2023-03-25T04:33:38.333467+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117119,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:36:50 2023","interfaces":[{"interface":"back","average":{"1min":0.254,"5min":0.25600000000000001,"15min":0.27900000000000003},"min":{"1min":0.20499999999999999,"5min":0.16,"15min":0.16},"max":{"1min":0.371,"5min":0.42599999999999999,"15min":0.45100000000000001},"last":0.23000000000000001},{"interface":"front","average":{"1min":0.26800000000000002,"5min":0.26500000000000001,"15min":0.28399999999999997},"min":{"1min":0.19700000000000001,"5min":0.17999999999999999,"15min":0.17499999999999999},"max":{"1min":0.34699999999999998,"5min":0.45200000000000001,"15min":0.46300000000000002},"last":0.20899999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051710,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:36:44 2023","interfaces":[{"interface":"back","average":{"1min":0.28399999999999997,"5min":0.27700000000000002,"15min":0.26100000000000001},"min":{"1min":0.192,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.40899999999999997,"5min":0.42699999999999999,"15min":0.42699999999999999},"last":0.39700000000000002},{"interface":"front","average":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.27500000000000002},"min":{"1min":0.182,"5min":0.182,"15min":0.16600000000000001},"max":{"1min":0.439,"5min":0.45800000000000002,"15min":0.45800000000000002},"last":0.439}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:37:42.400+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)

Ignoring malparsed backtrace: 2023-03-25T04:38:04.766+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:38:04.922 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:38:05.246 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:38:19.605 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":411,"stamp":"2023-03-25T04:38:18.220966+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002368"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":38,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:36:03.347070+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:36:03.347070+0000","last_peered":"2023-03-25T04:36:03.347070+0000","last_clean":"2023-03-25T04:36:03.347070+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:36:03.347070+0000","last_undegraded":"2023-03-25T04:36:03.347070+0000","last_fullsized":"2023-03-25T04:36:03.347070+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:36:03.347025+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:36:03.347025+0000","last_peered":"2023-03-25T04:36:03.347025+0000","last_clean":"2023-03-25T04:36:03.347025+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:36:03.347025+0000","last_undegraded":"2023-03-25T04:36:03.347025+0000","last_fullsized":"2023-03-25T04:36:03.347025+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:23.343585+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:35:23.343585+0000","last_peered":"2023-03-25T04:35:23.343585+0000","last_clean":"2023-03-25T04:35:23.343585+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:35:23.343585+0000","last_undegraded":"2023-03-25T04:35:23.343585+0000","last_fullsized":"2023-03-25T04:35:23.343585+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:35.587025+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:35:35.587025+0000","last_peered":"2023-03-25T04:35:35.587025+0000","last_clean":"2023-03-25T04:35:35.587025+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:35:35.587025+0000","last_undegraded":"2023-03-25T04:35:35.587025+0000","last_fullsized":"2023-03-25T04:35:35.587025+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":38,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:43.345308+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:35:43.345308+0000","last_peered":"2023-03-25T04:35:43.345308+0000","last_clean":"2023-03-25T04:35:43.345308+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:35:43.345308+0000","last_undegraded":"2023-03-25T04:35:43.345308+0000","last_fullsized":"2023-03-25T04:35:43.345308+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:43.345324+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:35:43.345324+0000","last_peered":"2023-03-25T04:35:43.345324+0000","last_clean":"2023-03-25T04:35:43.345324+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:35:43.345324+0000","last_undegraded":"2023-03-25T04:35:43.345324+0000","last_fullsized":"2023-03-25T04:35:43.345324+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117127,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:37:53 2023","interfaces":[{"interface":"back","average":{"1min":0.25800000000000001,"5min":0.253,"15min":0.27700000000000002},"min":{"1min":0.184,"5min":0.16,"15min":0.16},"max":{"1min":0.44600000000000001,"5min":0.44600000000000001,"15min":0.45100000000000001},"last":0.35199999999999998},{"interface":"front","average":{"1min":0.26900000000000002,"5min":0.26500000000000001,"15min":0.28199999999999997},"min":{"1min":0.192,"5min":0.17999999999999999,"15min":0.17499999999999999},"max":{"1min":0.46400000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"last":0.33200000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051717,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:37:44 2023","interfaces":[{"interface":"back","average":{"1min":0.27300000000000002,"5min":0.28199999999999997,"15min":0.26200000000000001},"min":{"1min":0.17899999999999999,"5min":0.17299999999999999,"15min":0.17000000000000001},"max":{"1min":0.41799999999999998,"5min":0.42099999999999999,"15min":0.42699999999999999},"last":0.28699999999999998},{"interface":"front","average":{"1min":0.29399999999999998,"5min":0.29099999999999998,"15min":0.27700000000000002},"min":{"1min":0.19900000000000001,"5min":0.182,"15min":0.16600000000000001},"max":{"1min":0.441,"5min":0.441,"15min":0.45800000000000002},"last":0.26300000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:38:19.718+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:38:20.315+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)

Ignoring malparsed backtrace: 2023-03-25T04:38:44.283 INFO:teuthology.orchestra.run.smithi070.stdout:
, 2023-03-25T04:38:44.283 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":781,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":16,"modified":"2023-03-25T04:38:40.225321+0000","services":{}},"progress_events":{}}
, 2023-03-25T04:38:44.296 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now

Ignoring malparsed backtrace: 2023-03-25T04:39:02.477+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:39:02.578+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:39:03.524+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:39:20.359 INFO:teuthology.orchestra.run.smithi070.stdout:
, 2023-03-25T04:39:20.359 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":818,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":16,"modified":"2023-03-25T04:38:40.225321+0000","services":{}},"progress_events":{}}
, 2023-03-25T04:39:20.372 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now

Ignoring malparsed backtrace: 2023-03-25T04:39:37.527 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:39:37.541 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":450,"stamp":"2023-03-25T04:39:36.235332+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002086"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":38,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:36:03.347070+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:36:03.347070+0000","last_peered":"2023-03-25T04:36:03.347070+0000","last_clean":"2023-03-25T04:36:03.347070+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:36:03.347070+0000","last_undegraded":"2023-03-25T04:36:03.347070+0000","last_fullsized":"2023-03-25T04:36:03.347070+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:36:03.347025+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:36:03.347025+0000","last_peered":"2023-03-25T04:36:03.347025+0000","last_clean":"2023-03-25T04:36:03.347025+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:36:03.347025+0000","last_undegraded":"2023-03-25T04:36:03.347025+0000","last_fullsized":"2023-03-25T04:36:03.347025+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:23.353778+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:37:23.353778+0000","last_peered":"2023-03-25T04:37:23.353778+0000","last_clean":"2023-03-25T04:37:23.353778+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:37:23.353778+0000","last_undegraded":"2023-03-25T04:37:23.353778+0000","last_fullsized":"2023-03-25T04:37:23.353778+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:35.587025+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:35:35.587025+0000","last_peered":"2023-03-25T04:35:35.587025+0000","last_clean":"2023-03-25T04:35:35.587025+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:35:35.587025+0000","last_undegraded":"2023-03-25T04:35:35.587025+0000","last_fullsized":"2023-03-25T04:35:35.587025+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":38,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:43.345308+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:35:43.345308+0000","last_peered":"2023-03-25T04:35:43.345308+0000","last_clean":"2023-03-25T04:35:43.345308+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:35:43.345308+0000","last_undegraded":"2023-03-25T04:35:43.345308+0000","last_fullsized":"2023-03-25T04:35:43.345308+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:43.345324+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:35:43.345324+0000","last_peered":"2023-03-25T04:35:43.345324+0000","last_clean":"2023-03-25T04:35:43.345324+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:35:43.345324+0000","last_undegraded":"2023-03-25T04:35:43.345324+0000","last_fullsized":"2023-03-25T04:35:43.345324+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117143,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:38:54 2023","interfaces":[{"interface":"back","average":{"1min":0.25800000000000001,"5min":0.25700000000000001,"15min":0.27400000000000002},"min":{"1min":0.192,"5min":0.17399999999999999,"15min":0.16},"max":{"1min":0.39400000000000002,"5min":0.44600000000000001,"15min":0.45100000000000001},"last":0.26500000000000001},{"interface":"front","average":{"1min":0.25900000000000001,"5min":0.26700000000000002,"15min":0.27900000000000003},"min":{"1min":0.19500000000000001,"5min":0.188,"15min":0.17499999999999999},"max":{"1min":0.41499999999999998,"5min":0.46400000000000002,"15min":0.46400000000000002},"last":0.29099999999999998}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051733,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:38:45 2023","interfaces":[{"interface":"back","average":{"1min":0.27300000000000002,"5min":0.28100000000000003,"15min":0.26400000000000001},"min":{"1min":0.19400000000000001,"5min":0.17299999999999999,"15min":0.17000000000000001},"max":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"last":0.27300000000000002},{"interface":"front","average":{"1min":0.28799999999999998,"5min":0.29099999999999998,"15min":0.27800000000000002},"min":{"1min":0.19800000000000001,"5min":0.182,"15min":0.16600000000000001},"max":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"last":0.308}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:39:54.416+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:39:54.455 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:39:54.780 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:40:09.288+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:40:09.389+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:40:10.267+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)

Ignoring malparsed backtrace: 2023-03-25T04:40:28.984 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":476,"stamp":"2023-03-25T04:40:28.245911+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002693"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":39,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:38:13.358968+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:38:13.358968+0000","last_peered":"2023-03-25T04:38:13.358968+0000","last_clean":"2023-03-25T04:38:13.358968+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:38:13.358968+0000","last_undegraded":"2023-03-25T04:38:13.358968+0000","last_fullsized":"2023-03-25T04:38:13.358968+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:38:13.358922+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:38:13.358922+0000","last_peered":"2023-03-25T04:38:13.358922+0000","last_clean":"2023-03-25T04:38:13.358922+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:38:13.358922+0000","last_undegraded":"2023-03-25T04:38:13.358922+0000","last_fullsized":"2023-03-25T04:38:13.358922+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:23.353778+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:37:23.353778+0000","last_peered":"2023-03-25T04:37:23.353778+0000","last_clean":"2023-03-25T04:37:23.353778+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:37:23.353778+0000","last_undegraded":"2023-03-25T04:37:23.353778+0000","last_fullsized":"2023-03-25T04:37:23.353778+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:40.598579+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:37:40.598579+0000","last_peered":"2023-03-25T04:37:40.598579+0000","last_clean":"2023-03-25T04:37:40.598579+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:37:40.598579+0000","last_undegraded":"2023-03-25T04:37:40.598579+0000","last_fullsized":"2023-03-25T04:37:40.598579+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":39,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:48.356344+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:37:48.356344+0000","last_peered":"2023-03-25T04:37:48.356344+0000","last_clean":"2023-03-25T04:37:48.356344+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:37:48.356344+0000","last_undegraded":"2023-03-25T04:37:48.356344+0000","last_fullsized":"2023-03-25T04:37:48.356344+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:48.356365+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:37:48.356365+0000","last_peered":"2023-03-25T04:37:48.356365+0000","last_clean":"2023-03-25T04:37:48.356365+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:37:48.356365+0000","last_undegraded":"2023-03-25T04:37:48.356365+0000","last_fullsized":"2023-03-25T04:37:48.356365+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117153,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:39:54 2023","interfaces":[{"interface":"back","average":{"1min":0.25900000000000001,"5min":0.25600000000000001,"15min":0.27100000000000002},"min":{"1min":0.188,"5min":0.17899999999999999,"15min":0.16},"max":{"1min":0.41999999999999998,"5min":0.44600000000000001,"15min":0.45100000000000001},"last":0.255},{"interface":"front","average":{"1min":0.27500000000000002,"5min":0.26600000000000001,"15min":0.27700000000000002},"min":{"1min":0.17699999999999999,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.45600000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"last":0.26800000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051743,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:39:47 2023","interfaces":[{"interface":"back","average":{"1min":0.26300000000000001,"5min":0.27700000000000002,"15min":0.26500000000000001},"min":{"1min":0.187,"5min":0.17299999999999999,"15min":0.17000000000000001},"max":{"1min":0.433,"5min":0.433,"15min":0.433},"last":0.36599999999999999},{"interface":"front","average":{"1min":0.27100000000000002,"5min":0.28799999999999998,"15min":0.27900000000000003},"min":{"1min":0.20100000000000001,"5min":0.182,"15min":0.16600000000000001},"max":{"1min":0.48599999999999999,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.34999999999999998}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:40:28.986 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean
, 2023-03-25T04:40:28.986 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}

Ignoring malparsed backtrace: 2023-03-25T04:40:47.229+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:40:47.503+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:40:48.251+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)

Ignoring malparsed backtrace: 2023-03-25T04:41:03.267 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:41:03.578 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all

Ignoring malparsed backtrace: 2023-03-25T04:41:21.347 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":502,"stamp":"2023-03-25T04:41:20.256204+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002342"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":39,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:38:13.358968+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:38:13.358968+0000","last_peered":"2023-03-25T04:38:13.358968+0000","last_clean":"2023-03-25T04:38:13.358968+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:38:13.358968+0000","last_undegraded":"2023-03-25T04:38:13.358968+0000","last_fullsized":"2023-03-25T04:38:13.358968+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:38:13.358922+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:38:13.358922+0000","last_peered":"2023-03-25T04:38:13.358922+0000","last_clean":"2023-03-25T04:38:13.358922+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:38:13.358922+0000","last_undegraded":"2023-03-25T04:38:13.358922+0000","last_fullsized":"2023-03-25T04:38:13.358922+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:23.353778+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:37:23.353778+0000","last_peered":"2023-03-25T04:37:23.353778+0000","last_clean":"2023-03-25T04:37:23.353778+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:37:23.353778+0000","last_undegraded":"2023-03-25T04:37:23.353778+0000","last_fullsized":"2023-03-25T04:37:23.353778+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:40.598579+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:37:40.598579+0000","last_peered":"2023-03-25T04:37:40.598579+0000","last_clean":"2023-03-25T04:37:40.598579+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:37:40.598579+0000","last_undegraded":"2023-03-25T04:37:40.598579+0000","last_fullsized":"2023-03-25T04:37:40.598579+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":39,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:48.356344+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:37:48.356344+0000","last_peered":"2023-03-25T04:37:48.356344+0000","last_clean":"2023-03-25T04:37:48.356344+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:37:48.356344+0000","last_undegraded":"2023-03-25T04:37:48.356344+0000","last_fullsized":"2023-03-25T04:37:48.356344+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:48.356365+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:37:48.356365+0000","last_peered":"2023-03-25T04:37:48.356365+0000","last_clean":"2023-03-25T04:37:48.356365+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:37:48.356365+0000","last_undegraded":"2023-03-25T04:37:48.356365+0000","last_fullsized":"2023-03-25T04:37:48.356365+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117163,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:40:56 2023","interfaces":[{"interface":"back","average":{"1min":0.28799999999999998,"5min":0.26300000000000001,"15min":0.27100000000000002},"min":{"1min":0.188,"5min":0.184,"15min":0.16},"max":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.17499999999999999},{"interface":"front","average":{"1min":0.29599999999999999,"5min":0.27300000000000002,"15min":0.27700000000000002},"min":{"1min":0.20599999999999999,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.43099999999999999,"5min":0.46400000000000002,"15min":0.46400000000000002},"last":0.20699999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051754,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:40:52 2023","interfaces":[{"interface":"back","average":{"1min":0.28599999999999998,"5min":0.27500000000000002,"15min":0.26700000000000002},"min":{"1min":0.17899999999999999,"5min":0.17899999999999999,"15min":0.17000000000000001},"max":{"1min":0.44,"5min":0.44,"15min":0.44},"last":0.23699999999999999},{"interface":"front","average":{"1min":0.29799999999999999,"5min":0.28699999999999998,"15min":0.28100000000000003},"min":{"1min":0.20300000000000001,"5min":0.182,"15min":0.16600000000000001},"max":{"1min":0.47599999999999998,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.28000000000000003}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:41:21.349 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean
, 2023-03-25T04:41:21.349 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}

Ignoring malparsed backtrace: 2023-03-25T04:41:39.867+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:41:40.383+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:41:40.905+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:41:58.219 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json
, 2023-03-25T04:41:58.319+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:41:58.644 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:42:18.193 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":530,"stamp":"2023-03-25T04:42:16.266659+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002356"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":39,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:38:13.358968+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:38:13.358968+0000","last_peered":"2023-03-25T04:38:13.358968+0000","last_clean":"2023-03-25T04:38:13.358968+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:38:13.358968+0000","last_undegraded":"2023-03-25T04:38:13.358968+0000","last_fullsized":"2023-03-25T04:38:13.358968+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:38:13.358922+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:38:13.358922+0000","last_peered":"2023-03-25T04:38:13.358922+0000","last_clean":"2023-03-25T04:38:13.358922+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:38:13.358922+0000","last_undegraded":"2023-03-25T04:38:13.358922+0000","last_fullsized":"2023-03-25T04:38:13.358922+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:23.364593+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:39:23.364593+0000","last_peered":"2023-03-25T04:39:23.364593+0000","last_clean":"2023-03-25T04:39:23.364593+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:39:23.364593+0000","last_undegraded":"2023-03-25T04:39:23.364593+0000","last_fullsized":"2023-03-25T04:39:23.364593+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:45.608475+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:39:45.608475+0000","last_peered":"2023-03-25T04:39:45.608475+0000","last_clean":"2023-03-25T04:39:45.608475+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:39:45.608475+0000","last_undegraded":"2023-03-25T04:39:45.608475+0000","last_fullsized":"2023-03-25T04:39:45.608475+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":40,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:53.367398+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:39:53.367398+0000","last_peered":"2023-03-25T04:39:53.367398+0000","last_clean":"2023-03-25T04:39:53.367398+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:39:53.367398+0000","last_undegraded":"2023-03-25T04:39:53.367398+0000","last_fullsized":"2023-03-25T04:39:53.367398+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:53.367424+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:39:53.367424+0000","last_peered":"2023-03-25T04:39:53.367424+0000","last_clean":"2023-03-25T04:39:53.367424+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:39:53.367424+0000","last_undegraded":"2023-03-25T04:39:53.367424+0000","last_fullsized":"2023-03-25T04:39:53.367424+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117175,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:41:58 2023","interfaces":[{"interface":"back","average":{"1min":0.25800000000000001,"5min":0.26400000000000001,"15min":0.26800000000000002},"min":{"1min":0.17499999999999999,"5min":0.17499999999999999,"15min":0.16},"max":{"1min":0.39100000000000001,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.22900000000000001},{"interface":"front","average":{"1min":0.26300000000000001,"5min":0.27200000000000002,"15min":0.27400000000000002},"min":{"1min":0.20599999999999999,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.35699999999999998,"5min":0.46400000000000002,"15min":0.46400000000000002},"last":0.26400000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051765,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:41:52 2023","interfaces":[{"interface":"back","average":{"1min":0.26500000000000001,"5min":0.27200000000000002,"15min":0.26800000000000002},"min":{"1min":0.17000000000000001,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.39200000000000002,"5min":0.44,"15min":0.44},"last":0.23899999999999999},{"interface":"front","average":{"1min":0.28100000000000003,"5min":0.28599999999999998,"15min":0.28199999999999997},"min":{"1min":0.19400000000000001,"5min":0.19400000000000001,"15min":0.16600000000000001},"max":{"1min":0.41399999999999998,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.254}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:42:18.297+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:42:18.958+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:42:35.512 INFO:teuthology.orchestra.run.smithi070.stdout:
, 2023-03-25T04:42:35.512 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":1013,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":17,"modified":"2023-03-25T04:41:54.262744+0000","services":{}},"progress_events":{}}
, 2023-03-25T04:42:35.525 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:42:52.741 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":548,"stamp":"2023-03-25T04:42:52.273601+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002221"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":40,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:40:23.370056+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:40:23.370056+0000","last_peered":"2023-03-25T04:40:23.370056+0000","last_clean":"2023-03-25T04:40:23.370056+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:40:23.370056+0000","last_undegraded":"2023-03-25T04:40:23.370056+0000","last_fullsized":"2023-03-25T04:40:23.370056+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:40:23.370039+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:40:23.370039+0000","last_peered":"2023-03-25T04:40:23.370039+0000","last_clean":"2023-03-25T04:40:23.370039+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:40:23.370039+0000","last_undegraded":"2023-03-25T04:40:23.370039+0000","last_fullsized":"2023-03-25T04:40:23.370039+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:23.364593+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:39:23.364593+0000","last_peered":"2023-03-25T04:39:23.364593+0000","last_clean":"2023-03-25T04:39:23.364593+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:39:23.364593+0000","last_undegraded":"2023-03-25T04:39:23.364593+0000","last_fullsized":"2023-03-25T04:39:23.364593+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:45.608475+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:39:45.608475+0000","last_peered":"2023-03-25T04:39:45.608475+0000","last_clean":"2023-03-25T04:39:45.608475+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:39:45.608475+0000","last_undegraded":"2023-03-25T04:39:45.608475+0000","last_fullsized":"2023-03-25T04:39:45.608475+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":40,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:53.367398+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:39:53.367398+0000","last_peered":"2023-03-25T04:39:53.367398+0000","last_clean":"2023-03-25T04:39:53.367398+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:39:53.367398+0000","last_undegraded":"2023-03-25T04:39:53.367398+0000","last_fullsized":"2023-03-25T04:39:53.367398+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:53.367424+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:39:53.367424+0000","last_peered":"2023-03-25T04:39:53.367424+0000","last_clean":"2023-03-25T04:39:53.367424+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:39:53.367424+0000","last_undegraded":"2023-03-25T04:39:53.367424+0000","last_fullsized":"2023-03-25T04:39:53.367424+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117182,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:41:58 2023","interfaces":[{"interface":"back","average":{"1min":0.25800000000000001,"5min":0.26400000000000001,"15min":0.26800000000000002},"min":{"1min":0.17499999999999999,"5min":0.17499999999999999,"15min":0.16},"max":{"1min":0.39100000000000001,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.19600000000000001},{"interface":"front","average":{"1min":0.26300000000000001,"5min":0.27200000000000002,"15min":0.27400000000000002},"min":{"1min":0.20599999999999999,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.35699999999999998,"5min":0.46400000000000002,"15min":0.46400000000000002},"last":0.219}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051773,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:41:52 2023","interfaces":[{"interface":"back","average":{"1min":0.26500000000000001,"5min":0.27200000000000002,"15min":0.26800000000000002},"min":{"1min":0.17000000000000001,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.39200000000000002,"5min":0.44,"15min":0.44},"last":0.19},{"interface":"front","average":{"1min":0.28100000000000003,"5min":0.28599999999999998,"15min":0.28199999999999997},"min":{"1min":0.19400000000000001,"5min":0.19400000000000001,"15min":0.16600000000000001},"max":{"1min":0.41399999999999998,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.17599999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:42:52.742 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean

Ignoring malparsed backtrace: 2023-03-25T04:43:09.973 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:43:09.986 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":556,"stamp":"2023-03-25T04:43:08.276556+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002359"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":40,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:40:23.370056+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:40:23.370056+0000","last_peered":"2023-03-25T04:40:23.370056+0000","last_clean":"2023-03-25T04:40:23.370056+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:40:23.370056+0000","last_undegraded":"2023-03-25T04:40:23.370056+0000","last_fullsized":"2023-03-25T04:40:23.370056+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:40:23.370039+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:40:23.370039+0000","last_peered":"2023-03-25T04:40:23.370039+0000","last_clean":"2023-03-25T04:40:23.370039+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:40:23.370039+0000","last_undegraded":"2023-03-25T04:40:23.370039+0000","last_fullsized":"2023-03-25T04:40:23.370039+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:23.364593+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:39:23.364593+0000","last_peered":"2023-03-25T04:39:23.364593+0000","last_clean":"2023-03-25T04:39:23.364593+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:39:23.364593+0000","last_undegraded":"2023-03-25T04:39:23.364593+0000","last_fullsized":"2023-03-25T04:39:23.364593+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:45.608475+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:39:45.608475+0000","last_peered":"2023-03-25T04:39:45.608475+0000","last_clean":"2023-03-25T04:39:45.608475+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:39:45.608475+0000","last_undegraded":"2023-03-25T04:39:45.608475+0000","last_fullsized":"2023-03-25T04:39:45.608475+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":40,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:53.367398+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:39:53.367398+0000","last_peered":"2023-03-25T04:39:53.367398+0000","last_clean":"2023-03-25T04:39:53.367398+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:39:53.367398+0000","last_undegraded":"2023-03-25T04:39:53.367398+0000","last_fullsized":"2023-03-25T04:39:53.367398+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:53.367424+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:39:53.367424+0000","last_peered":"2023-03-25T04:39:53.367424+0000","last_clean":"2023-03-25T04:39:53.367424+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:39:53.367424+0000","last_undegraded":"2023-03-25T04:39:53.367424+0000","last_fullsized":"2023-03-25T04:39:53.367424+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117185,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:43:00 2023","interfaces":[{"interface":"back","average":{"1min":0.27400000000000002,"5min":0.26700000000000002,"15min":0.26600000000000001},"min":{"1min":0.19600000000000001,"5min":0.17499999999999999,"15min":0.16},"max":{"1min":0.42999999999999999,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.27800000000000002},{"interface":"front","average":{"1min":0.28199999999999997,"5min":0.27500000000000002,"15min":0.27300000000000002},"min":{"1min":0.185,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.47199999999999998,"5min":0.47199999999999998,"15min":0.47199999999999998},"last":0.247}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051776,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:42:54 2023","interfaces":[{"interface":"back","average":{"1min":0.26300000000000001,"5min":0.27000000000000002,"15min":0.26900000000000002},"min":{"1min":0.17399999999999999,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.438,"5min":0.44,"15min":0.44},"last":0.20100000000000001},{"interface":"front","average":{"1min":0.27400000000000002,"5min":0.28199999999999997,"15min":0.28299999999999997},"min":{"1min":0.17599999999999999,"5min":0.17599999999999999,"15min":0.16600000000000001},"max":{"1min":0.41599999999999998,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.23499999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:43:25.042+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:43:25.411+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:43:26.013+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:43:47.937+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:43:48.534+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:43:48.889+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:44:09.318 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:44:09.331 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":586,"stamp":"2023-03-25T04:44:08.288933+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002697"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":40,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:40:23.370056+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:40:23.370056+0000","last_peered":"2023-03-25T04:40:23.370056+0000","last_clean":"2023-03-25T04:40:23.370056+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:40:23.370056+0000","last_undegraded":"2023-03-25T04:40:23.370056+0000","last_fullsized":"2023-03-25T04:40:23.370056+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:40:23.370039+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:40:23.370039+0000","last_peered":"2023-03-25T04:40:23.370039+0000","last_clean":"2023-03-25T04:40:23.370039+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:40:23.370039+0000","last_undegraded":"2023-03-25T04:40:23.370039+0000","last_fullsized":"2023-03-25T04:40:23.370039+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:23.375106+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:41:23.375106+0000","last_peered":"2023-03-25T04:41:23.375106+0000","last_clean":"2023-03-25T04:41:23.375106+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:41:23.375106+0000","last_undegraded":"2023-03-25T04:41:23.375106+0000","last_fullsized":"2023-03-25T04:41:23.375106+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:50.618930+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:41:50.618930+0000","last_peered":"2023-03-25T04:41:50.618930+0000","last_clean":"2023-03-25T04:41:50.618930+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:41:50.618930+0000","last_undegraded":"2023-03-25T04:41:50.618930+0000","last_fullsized":"2023-03-25T04:41:50.618930+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":41,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:58.377821+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:41:58.377821+0000","last_peered":"2023-03-25T04:41:58.377821+0000","last_clean":"2023-03-25T04:41:58.377821+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:41:58.377821+0000","last_undegraded":"2023-03-25T04:41:58.377821+0000","last_fullsized":"2023-03-25T04:41:58.377821+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:58.377836+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:41:58.377836+0000","last_peered":"2023-03-25T04:41:58.377836+0000","last_clean":"2023-03-25T04:41:58.377836+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:41:58.377836+0000","last_undegraded":"2023-03-25T04:41:58.377836+0000","last_fullsized":"2023-03-25T04:41:58.377836+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117197,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:44:01 2023","interfaces":[{"interface":"back","average":{"1min":0.27200000000000002,"5min":0.27000000000000002,"15min":0.26400000000000001},"min":{"1min":0.20399999999999999,"5min":0.17499999999999999,"15min":0.16},"max":{"1min":0.38400000000000001,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.245},{"interface":"front","average":{"1min":0.27600000000000002,"5min":0.27800000000000002,"15min":0.27100000000000002},"min":{"1min":0.23100000000000001,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.40400000000000003,"5min":0.47199999999999998,"15min":0.47199999999999998},"last":0.27500000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051788,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:43:55 2023","interfaces":[{"interface":"back","average":{"1min":0.26100000000000001,"5min":0.26700000000000002,"15min":0.27000000000000002},"min":{"1min":0.17799999999999999,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.374,"5min":0.44,"15min":0.44},"last":0.30299999999999999},{"interface":"front","average":{"1min":0.28299999999999997,"5min":0.28100000000000003,"15min":0.28399999999999997},"min":{"1min":0.20899999999999999,"5min":0.17599999999999999,"15min":0.16600000000000001},"max":{"1min":0.40100000000000002,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.27800000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:44:28.923 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":1126,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":19,"modified":"2023-03-25T04:43:20.278959+0000","services":{}},"progress_events":{}}
, 2023-03-25T04:44:28.936 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now
, 2023-03-25T04:44:28.937 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace: 2023-03-25T04:44:48.786 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:44:48.800 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":606,"stamp":"2023-03-25T04:44:48.296805+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002182"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":41,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:42:34.382196+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:42:34.382196+0000","last_peered":"2023-03-25T04:42:34.382196+0000","last_clean":"2023-03-25T04:42:34.382196+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:42:34.382196+0000","last_undegraded":"2023-03-25T04:42:34.382196+0000","last_fullsized":"2023-03-25T04:42:34.382196+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:42:29.381744+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:42:29.381744+0000","last_peered":"2023-03-25T04:42:29.381744+0000","last_clean":"2023-03-25T04:42:29.381744+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:42:29.381744+0000","last_undegraded":"2023-03-25T04:42:29.381744+0000","last_fullsized":"2023-03-25T04:42:29.381744+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:23.375106+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:41:23.375106+0000","last_peered":"2023-03-25T04:41:23.375106+0000","last_clean":"2023-03-25T04:41:23.375106+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:41:23.375106+0000","last_undegraded":"2023-03-25T04:41:23.375106+0000","last_fullsized":"2023-03-25T04:41:23.375106+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:50.618930+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:41:50.618930+0000","last_peered":"2023-03-25T04:41:50.618930+0000","last_clean":"2023-03-25T04:41:50.618930+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:41:50.618930+0000","last_undegraded":"2023-03-25T04:41:50.618930+0000","last_fullsized":"2023-03-25T04:41:50.618930+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":41,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:58.377821+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:41:58.377821+0000","last_peered":"2023-03-25T04:41:58.377821+0000","last_clean":"2023-03-25T04:41:58.377821+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:41:58.377821+0000","last_undegraded":"2023-03-25T04:41:58.377821+0000","last_fullsized":"2023-03-25T04:41:58.377821+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:58.377836+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:41:58.377836+0000","last_peered":"2023-03-25T04:41:58.377836+0000","last_clean":"2023-03-25T04:41:58.377836+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:41:58.377836+0000","last_undegraded":"2023-03-25T04:41:58.377836+0000","last_fullsized":"2023-03-25T04:41:58.377836+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117206,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:44:01 2023","interfaces":[{"interface":"back","average":{"1min":0.27200000000000002,"5min":0.27000000000000002,"15min":0.26400000000000001},"min":{"1min":0.20399999999999999,"5min":0.17499999999999999,"15min":0.16},"max":{"1min":0.38400000000000001,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.216},{"interface":"front","average":{"1min":0.27600000000000002,"5min":0.27800000000000002,"15min":0.27100000000000002},"min":{"1min":0.23100000000000001,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.40400000000000003,"5min":0.47199999999999998,"15min":0.47199999999999998},"last":0.23999999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051796,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:43:55 2023","interfaces":[{"interface":"back","average":{"1min":0.26100000000000001,"5min":0.26700000000000002,"15min":0.27000000000000002},"min":{"1min":0.17799999999999999,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.374,"5min":0.44,"15min":0.44},"last":0.25600000000000001},{"interface":"front","average":{"1min":0.28299999999999997,"5min":0.28100000000000003,"15min":0.28399999999999997},"min":{"1min":0.20899999999999999,"5min":0.17599999999999999,"15min":0.16600000000000001},"max":{"1min":0.40100000000000002,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.28000000000000003}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:45:05.762 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:45:06.073 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all

Ignoring malparsed backtrace: 2023-03-25T04:45:25.658 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":624,"stamp":"2023-03-25T04:45:24.304294+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002366"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":41,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:42:34.382196+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:42:34.382196+0000","last_peered":"2023-03-25T04:42:34.382196+0000","last_clean":"2023-03-25T04:42:34.382196+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:42:34.382196+0000","last_undegraded":"2023-03-25T04:42:34.382196+0000","last_fullsized":"2023-03-25T04:42:34.382196+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:42:29.381744+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:42:29.381744+0000","last_peered":"2023-03-25T04:42:29.381744+0000","last_clean":"2023-03-25T04:42:29.381744+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:42:29.381744+0000","last_undegraded":"2023-03-25T04:42:29.381744+0000","last_fullsized":"2023-03-25T04:42:29.381744+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:23.375106+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:41:23.375106+0000","last_peered":"2023-03-25T04:41:23.375106+0000","last_clean":"2023-03-25T04:41:23.375106+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:41:23.375106+0000","last_undegraded":"2023-03-25T04:41:23.375106+0000","last_fullsized":"2023-03-25T04:41:23.375106+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:50.618930+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:41:50.618930+0000","last_peered":"2023-03-25T04:41:50.618930+0000","last_clean":"2023-03-25T04:41:50.618930+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:41:50.618930+0000","last_undegraded":"2023-03-25T04:41:50.618930+0000","last_fullsized":"2023-03-25T04:41:50.618930+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":41,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:58.377821+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:41:58.377821+0000","last_peered":"2023-03-25T04:41:58.377821+0000","last_clean":"2023-03-25T04:41:58.377821+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:41:58.377821+0000","last_undegraded":"2023-03-25T04:41:58.377821+0000","last_fullsized":"2023-03-25T04:41:58.377821+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:58.377836+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:41:58.377836+0000","last_peered":"2023-03-25T04:41:58.377836+0000","last_clean":"2023-03-25T04:41:58.377836+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:41:58.377836+0000","last_undegraded":"2023-03-25T04:41:58.377836+0000","last_fullsized":"2023-03-25T04:41:58.377836+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117213,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:45:01 2023","interfaces":[{"interface":"back","average":{"1min":0.251,"5min":0.26800000000000002,"15min":0.26400000000000001},"min":{"1min":0.183,"5min":0.17499999999999999,"15min":0.16},"max":{"1min":0.41499999999999998,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.26900000000000002},{"interface":"front","average":{"1min":0.28000000000000003,"5min":0.27900000000000003,"15min":0.27100000000000002},"min":{"1min":0.20999999999999999,"5min":0.185,"15min":0.17499999999999999},"max":{"1min":0.44400000000000001,"5min":0.47199999999999998,"15min":0.47199999999999998},"last":0.27900000000000003}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051803,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:44:55 2023","interfaces":[{"interface":"back","average":{"1min":0.252,"5min":0.26500000000000001,"15min":0.26900000000000002},"min":{"1min":0.183,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.39700000000000002,"5min":0.44,"15min":0.44},"last":0.20699999999999999},{"interface":"front","average":{"1min":0.27000000000000002,"5min":0.28100000000000003,"15min":0.28299999999999997},"min":{"1min":0.20300000000000001,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":0.42099999999999999,"5min":0.47599999999999998,"15min":0.48599999999999999},"last":0.23599999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:45:25.659 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:45:25.969 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:45:42.735+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:45:42.744+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:45:42.897 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace: 2023-03-25T04:46:00.183 INFO:teuthology.orchestra.run.smithi070.stdout:
, 2023-03-25T04:46:00.183 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":1217,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":20,"modified":"2023-03-25T04:45:42.307700+0000","services":{"mgr":{"daemons":{"summary":"","x":{"start_epoch":0,"start_stamp":"0.000000","gid":0,"addr":"(unrecognized address family 0)/0","metadata":{},"task_status":{}}}}}},"progress_events":{}}
, 2023-03-25T04:46:00.197 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now

Ignoring malparsed backtrace: 2023-03-25T04:46:19.910+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:46:20.039 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all

Ignoring malparsed backtrace: 2023-03-25T04:46:39.611 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}
, 2023-03-25T04:46:39.611 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json
, 2023-03-25T04:46:39.613+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:46:58.485+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:46:58.764+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:46:59.519+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:47:14.132 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:47:14.146 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":678,"stamp":"2023-03-25T04:47:12.325044+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002571"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":42,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:44.393426+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:44:44.393426+0000","last_peered":"2023-03-25T04:44:44.393426+0000","last_clean":"2023-03-25T04:44:44.393426+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:44:44.393426+0000","last_undegraded":"2023-03-25T04:44:44.393426+0000","last_fullsized":"2023-03-25T04:44:44.393426+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:39.393044+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:44:39.393044+0000","last_peered":"2023-03-25T04:44:39.393044+0000","last_clean":"2023-03-25T04:44:39.393044+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:44:39.393044+0000","last_undegraded":"2023-03-25T04:44:39.393044+0000","last_fullsized":"2023-03-25T04:44:39.393044+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:43:24.386177+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:43:24.386177+0000","last_peered":"2023-03-25T04:43:24.386177+0000","last_clean":"2023-03-25T04:43:24.386177+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:43:24.386177+0000","last_undegraded":"2023-03-25T04:43:24.386177+0000","last_fullsized":"2023-03-25T04:43:24.386177+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:43:55.629798+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:43:55.629798+0000","last_peered":"2023-03-25T04:43:55.629798+0000","last_clean":"2023-03-25T04:43:55.629798+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:43:55.629798+0000","last_undegraded":"2023-03-25T04:43:55.629798+0000","last_fullsized":"2023-03-25T04:43:55.629798+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":42,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:04.389904+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:44:04.389904+0000","last_peered":"2023-03-25T04:44:04.389904+0000","last_clean":"2023-03-25T04:44:04.389904+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:44:04.389904+0000","last_undegraded":"2023-03-25T04:44:04.389904+0000","last_fullsized":"2023-03-25T04:44:04.389904+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:04.389923+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:44:04.389923+0000","last_peered":"2023-03-25T04:44:04.389923+0000","last_clean":"2023-03-25T04:44:04.389923+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:44:04.389923+0000","last_undegraded":"2023-03-25T04:44:04.389923+0000","last_fullsized":"2023-03-25T04:44:04.389923+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117235,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:47:05 2023","interfaces":[{"interface":"back","average":{"1min":0.27800000000000002,"5min":0.27000000000000002,"15min":0.26300000000000001},"min":{"1min":0.20100000000000001,"5min":0.16900000000000001,"15min":0.16},"max":{"1min":0.39200000000000002,"5min":0.42999999999999999,"15min":0.45600000000000002},"last":0.379},{"interface":"front","average":{"1min":0.28399999999999997,"5min":0.28100000000000003,"15min":0.27200000000000002},"min":{"1min":0.17799999999999999,"5min":0.17799999999999999,"15min":0.17699999999999999},"max":{"1min":0.42599999999999999,"5min":0.47199999999999998,"15min":0.47199999999999998},"last":0.39800000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051825,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:47:01 2023","interfaces":[{"interface":"back","average":{"1min":0.27300000000000002,"5min":0.26100000000000001,"15min":0.27000000000000002},"min":{"1min":0.186,"5min":0.17399999999999999,"15min":0.17000000000000001},"max":{"1min":0.40400000000000003,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.23899999999999999},{"interface":"front","average":{"1min":0.28699999999999998,"5min":0.27500000000000002,"15min":0.28299999999999997},"min":{"1min":0.186,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.48599999999999999},"last":0.19600000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:47:31.198 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json
, 2023-03-25T04:47:31.411+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:47:31.789 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:47:49.509 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":696,"stamp":"2023-03-25T04:47:48.332093+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002697"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":42,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:44.393426+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:44:44.393426+0000","last_peered":"2023-03-25T04:44:44.393426+0000","last_clean":"2023-03-25T04:44:44.393426+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:44:44.393426+0000","last_undegraded":"2023-03-25T04:44:44.393426+0000","last_fullsized":"2023-03-25T04:44:44.393426+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:39.393044+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:44:39.393044+0000","last_peered":"2023-03-25T04:44:39.393044+0000","last_clean":"2023-03-25T04:44:39.393044+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:44:39.393044+0000","last_undegraded":"2023-03-25T04:44:39.393044+0000","last_fullsized":"2023-03-25T04:44:39.393044+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:45:24.396717+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:45:24.396717+0000","last_peered":"2023-03-25T04:45:24.396717+0000","last_clean":"2023-03-25T04:45:24.396717+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:45:24.396717+0000","last_undegraded":"2023-03-25T04:45:24.396717+0000","last_fullsized":"2023-03-25T04:45:24.396717+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:43:55.629798+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:43:55.629798+0000","last_peered":"2023-03-25T04:43:55.629798+0000","last_clean":"2023-03-25T04:43:55.629798+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:43:55.629798+0000","last_undegraded":"2023-03-25T04:43:55.629798+0000","last_fullsized":"2023-03-25T04:43:55.629798+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":42,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:04.389904+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:44:04.389904+0000","last_peered":"2023-03-25T04:44:04.389904+0000","last_clean":"2023-03-25T04:44:04.389904+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:44:04.389904+0000","last_undegraded":"2023-03-25T04:44:04.389904+0000","last_fullsized":"2023-03-25T04:44:04.389904+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:04.389923+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:44:04.389923+0000","last_peered":"2023-03-25T04:44:04.389923+0000","last_clean":"2023-03-25T04:44:04.389923+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:44:04.389923+0000","last_undegraded":"2023-03-25T04:44:04.389923+0000","last_fullsized":"2023-03-25T04:44:04.389923+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117242,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:47:05 2023","interfaces":[{"interface":"back","average":{"1min":0.27800000000000002,"5min":0.27000000000000002,"15min":0.26300000000000001},"min":{"1min":0.20100000000000001,"5min":0.16900000000000001,"15min":0.16},"max":{"1min":0.39200000000000002,"5min":0.42999999999999999,"15min":0.45600000000000002},"last":0.40500000000000003},{"interface":"front","average":{"1min":0.28399999999999997,"5min":0.28100000000000003,"15min":0.27200000000000002},"min":{"1min":0.17799999999999999,"5min":0.17799999999999999,"15min":0.17699999999999999},"max":{"1min":0.42599999999999999,"5min":0.47199999999999998,"15min":0.47199999999999998},"last":0.42399999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051834,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:47:01 2023","interfaces":[{"interface":"back","average":{"1min":0.27300000000000002,"5min":0.26100000000000001,"15min":0.27000000000000002},"min":{"1min":0.186,"5min":0.17399999999999999,"15min":0.17000000000000001},"max":{"1min":0.40400000000000003,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.25600000000000001},{"interface":"front","average":{"1min":0.28699999999999998,"5min":0.27500000000000002,"15min":0.28299999999999997},"min":{"1min":0.186,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.48599999999999999},"last":0.22800000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:47:49.510 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace:     ctx.managers[config['cluster']].wait_for_clean()
,   File "/home/teuthworker/src/github.com_ceph_ceph-c_511015c23eef0684a64171b710e432448df2b7c7/qa/tasks/ceph_manager.py", line 2747, in wait_for_clean
,     assert time.time() - start < timeout, \

Ignoring malparsed backtrace:  ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
,  1: /lib64/libpthread.so.0(+0x12cf0) [0x7f1453c2fcf0]
,  2: __pthread_mutex_lock()

Ignoring malparsed backtrace: 2023-03-25T04:28:05.172 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":142,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":5,"modified":"2023-03-25T04:27:12.092098+0000","services":{}},"progress_events":{}}
, 2023-03-25T04:28:05.185 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now
, 2023-03-25T04:28:05.186 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace: 2023-03-25T04:28:28.360 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":116,"stamp":"2023-03-25T04:28:28.108007+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002578"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.314661+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:27:45.314661+0000","last_peered":"2023-03-25T04:27:45.314661+0000","last_clean":"2023-03-25T04:27:45.314661+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:27:45.314661+0000","last_undegraded":"2023-03-25T04:27:45.314661+0000","last_fullsized":"2023-03-25T04:27:45.314661+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":25,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.310745+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:27:44.310745+0000","last_peered":"2023-03-25T04:27:44.310745+0000","last_clean":"2023-03-25T04:27:44.310745+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:27:44.310745+0000","last_undegraded":"2023-03-25T04:27:44.310745+0000","last_fullsized":"2023-03-25T04:27:44.310745+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":25,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.310650+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:27:44.310650+0000","last_peered":"2023-03-25T04:27:44.310650+0000","last_clean":"2023-03-25T04:27:44.310650+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:27:44.310650+0000","last_undegraded":"2023-03-25T04:27:44.310650+0000","last_fullsized":"2023-03-25T04:27:44.310650+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":26,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.312962+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:27:44.312962+0000","last_peered":"2023-03-25T04:27:44.312962+0000","last_clean":"2023-03-25T04:27:44.312962+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:27:44.312962+0000","last_undegraded":"2023-03-25T04:27:44.312962+0000","last_fullsized":"2023-03-25T04:27:44.312962+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315064+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:27:45.315064+0000","last_peered":"2023-03-25T04:27:45.315064+0000","last_clean":"2023-03-25T04:27:45.315064+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:27:45.315064+0000","last_undegraded":"2023-03-25T04:27:45.315064+0000","last_fullsized":"2023-03-25T04:27:45.315064+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":25,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.310707+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:27:44.310707+0000","last_peered":"2023-03-25T04:27:44.310707+0000","last_clean":"2023-03-25T04:27:44.310707+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:27:44.310707+0000","last_undegraded":"2023-03-25T04:27:44.310707+0000","last_fullsized":"2023-03-25T04:27:44.310707+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117007,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.375}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051597,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.186}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:28:28.362 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean
, 2023-03-25T04:28:28.362 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:28:45.655 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":124,"stamp":"2023-03-25T04:28:44.111265+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002492"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.314661+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:27:45.314661+0000","last_peered":"2023-03-25T04:27:45.314661+0000","last_clean":"2023-03-25T04:27:45.314661+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:27:45.314661+0000","last_undegraded":"2023-03-25T04:27:45.314661+0000","last_fullsized":"2023-03-25T04:27:45.314661+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":25,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.310745+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:27:44.310745+0000","last_peered":"2023-03-25T04:27:44.310745+0000","last_clean":"2023-03-25T04:27:44.310745+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:27:44.310745+0000","last_undegraded":"2023-03-25T04:27:44.310745+0000","last_fullsized":"2023-03-25T04:27:44.310745+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":25,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.310650+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:27:44.310650+0000","last_peered":"2023-03-25T04:27:44.310650+0000","last_clean":"2023-03-25T04:27:44.310650+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:27:44.310650+0000","last_undegraded":"2023-03-25T04:27:44.310650+0000","last_fullsized":"2023-03-25T04:27:44.310650+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":26,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.312962+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:27:44.312962+0000","last_peered":"2023-03-25T04:27:44.312962+0000","last_clean":"2023-03-25T04:27:44.312962+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:27:44.312962+0000","last_undegraded":"2023-03-25T04:27:44.312962+0000","last_fullsized":"2023-03-25T04:27:44.312962+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315064+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:27:45.315064+0000","last_peered":"2023-03-25T04:27:45.315064+0000","last_clean":"2023-03-25T04:27:45.315064+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:27:45.315064+0000","last_undegraded":"2023-03-25T04:27:45.315064+0000","last_fullsized":"2023-03-25T04:27:45.315064+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":25,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.310707+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:27:44.310707+0000","last_peered":"2023-03-25T04:27:44.310707+0000","last_clean":"2023-03-25T04:27:44.310707+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:27:44.310707+0000","last_undegraded":"2023-03-25T04:27:44.310707+0000","last_fullsized":"2023-03-25T04:27:44.310707+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117011,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:28:31 2023","interfaces":[{"interface":"back","average":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.36899999999999999},{"interface":"front","average":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"min":{"1min":0.19,"5min":0.19,"15min":0.19},"max":{"1min":0.46300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.39800000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051601,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:28:24 2023","interfaces":[{"interface":"back","average":{"1min":0.249,"5min":0.249,"15min":0.249},"min":{"1min":0.17699999999999999,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"last":0.254},{"interface":"front","average":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"min":{"1min":0.20200000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"last":0.312}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:28:45.656 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace: 2023-03-25T04:29:05.171 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}
, 2023-03-25T04:29:05.171 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json
, 2023-03-25T04:29:05.561 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:29:22.465 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:29:22.773 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all

Ignoring malparsed backtrace: 2023-03-25T04:29:42.405 INFO:teuthology.orchestra.run.smithi070.stdout:
, 2023-03-25T04:29:42.406 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":240,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":9,"modified":"2023-03-25T04:29:32.120358+0000","services":{"mon":{"daemons":{"summary":"","a":{"start_epoch":0,"start_stamp":"0.000000","gid":0,"addr":"(unrecognized address family 0)/0","metadata":{},"task_status":{}}}}}},"progress_events":{}}
, 2023-03-25T04:29:42.419 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:29:59.627 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":161,"stamp":"2023-03-25T04:29:58.125186+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002398"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315468+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:27:45.315468+0000","last_peered":"2023-03-25T04:27:45.315468+0000","last_clean":"2023-03-25T04:27:45.315468+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:27:45.315468+0000","last_undegraded":"2023-03-25T04:27:45.315468+0000","last_fullsized":"2023-03-25T04:27:45.315468+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312845+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:27:45.312845+0000","last_peered":"2023-03-25T04:27:45.312845+0000","last_clean":"2023-03-25T04:27:45.312845+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:27:45.312845+0000","last_undegraded":"2023-03-25T04:27:45.312845+0000","last_fullsized":"2023-03-25T04:27:45.312845+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312882+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:27:45.312882+0000","last_peered":"2023-03-25T04:27:45.312882+0000","last_clean":"2023-03-25T04:27:45.312882+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:27:45.312882+0000","last_undegraded":"2023-03-25T04:27:45.312882+0000","last_fullsized":"2023-03-25T04:27:45.312882+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.311730+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:27:45.311730+0000","last_peered":"2023-03-25T04:27:45.311730+0000","last_clean":"2023-03-25T04:27:45.311730+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:27:45.311730+0000","last_undegraded":"2023-03-25T04:27:45.311730+0000","last_fullsized":"2023-03-25T04:27:45.311730+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315735+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:27:45.315735+0000","last_peered":"2023-03-25T04:27:45.315735+0000","last_clean":"2023-03-25T04:27:45.315735+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:27:45.315735+0000","last_undegraded":"2023-03-25T04:27:45.315735+0000","last_fullsized":"2023-03-25T04:27:45.315735+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312833+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:27:45.312833+0000","last_peered":"2023-03-25T04:27:45.312833+0000","last_clean":"2023-03-25T04:27:45.312833+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:27:45.312833+0000","last_undegraded":"2023-03-25T04:27:45.312833+0000","last_fullsized":"2023-03-25T04:27:45.312833+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117026,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:29:33 2023","interfaces":[{"interface":"back","average":{"1min":0.26300000000000001,"5min":0.29099999999999998,"15min":0.29599999999999999},"min":{"1min":0.191,"5min":0.191,"15min":0.191},"max":{"1min":0.371,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.20499999999999999},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.29599999999999999,"15min":0.29999999999999999},"min":{"1min":0.21099999999999999,"5min":0.19,"15min":0.19},"max":{"1min":0.39800000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.19500000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051616,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:29:29 2023","interfaces":[{"interface":"back","average":{"1min":0.27200000000000002,"5min":0.253,"15min":0.25},"min":{"1min":0.183,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"last":0.26400000000000001},{"interface":"front","average":{"1min":0.28499999999999998,"5min":0.26800000000000002,"15min":0.26500000000000001},"min":{"1min":0.16600000000000001,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"last":0.29899999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:29:59.629 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean

Ignoring malparsed backtrace: 2023-03-25T04:30:16.889 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:30:16.903 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":170,"stamp":"2023-03-25T04:30:16.128427+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002222"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315468+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:27:45.315468+0000","last_peered":"2023-03-25T04:27:45.315468+0000","last_clean":"2023-03-25T04:27:45.315468+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:27:45.315468+0000","last_undegraded":"2023-03-25T04:27:45.315468+0000","last_fullsized":"2023-03-25T04:27:45.315468+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312845+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:27:45.312845+0000","last_peered":"2023-03-25T04:27:45.312845+0000","last_clean":"2023-03-25T04:27:45.312845+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:27:45.312845+0000","last_undegraded":"2023-03-25T04:27:45.312845+0000","last_fullsized":"2023-03-25T04:27:45.312845+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312882+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:27:45.312882+0000","last_peered":"2023-03-25T04:27:45.312882+0000","last_clean":"2023-03-25T04:27:45.312882+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:27:45.312882+0000","last_undegraded":"2023-03-25T04:27:45.312882+0000","last_fullsized":"2023-03-25T04:27:45.312882+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.311730+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:27:45.311730+0000","last_peered":"2023-03-25T04:27:45.311730+0000","last_clean":"2023-03-25T04:27:45.311730+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:27:45.311730+0000","last_undegraded":"2023-03-25T04:27:45.311730+0000","last_fullsized":"2023-03-25T04:27:45.311730+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315735+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:27:45.315735+0000","last_peered":"2023-03-25T04:27:45.315735+0000","last_clean":"2023-03-25T04:27:45.315735+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:27:45.315735+0000","last_undegraded":"2023-03-25T04:27:45.315735+0000","last_fullsized":"2023-03-25T04:27:45.315735+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312833+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:27:45.312833+0000","last_peered":"2023-03-25T04:27:45.312833+0000","last_clean":"2023-03-25T04:27:45.312833+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:27:45.312833+0000","last_undegraded":"2023-03-25T04:27:45.312833+0000","last_fullsized":"2023-03-25T04:27:45.312833+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117030,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:29:33 2023","interfaces":[{"interface":"back","average":{"1min":0.26300000000000001,"5min":0.29099999999999998,"15min":0.29599999999999999},"min":{"1min":0.191,"5min":0.191,"15min":0.191},"max":{"1min":0.371,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.245},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.29599999999999999,"15min":0.29999999999999999},"min":{"1min":0.21099999999999999,"5min":0.19,"15min":0.19},"max":{"1min":0.39800000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.20899999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051620,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:29:29 2023","interfaces":[{"interface":"back","average":{"1min":0.27200000000000002,"5min":0.253,"15min":0.25},"min":{"1min":0.183,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"last":0.19400000000000001},{"interface":"front","average":{"1min":0.28499999999999998,"5min":0.26800000000000002,"15min":0.26500000000000001},"min":{"1min":0.16600000000000001,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"last":0.23599999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:30:31.984+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:30:32.201+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:30:32.981+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)

Ignoring malparsed backtrace: 2023-03-25T04:30:48.449 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now
, 2023-03-25T04:30:48.450 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:30:48.760 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:31:08.342 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":196,"stamp":"2023-03-25T04:31:08.139018+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002431"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315468+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:27:45.315468+0000","last_peered":"2023-03-25T04:27:45.315468+0000","last_clean":"2023-03-25T04:27:45.315468+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:27:45.315468+0000","last_undegraded":"2023-03-25T04:27:45.315468+0000","last_fullsized":"2023-03-25T04:27:45.315468+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312845+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:27:45.312845+0000","last_peered":"2023-03-25T04:27:45.312845+0000","last_clean":"2023-03-25T04:27:45.312845+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:27:45.312845+0000","last_undegraded":"2023-03-25T04:27:45.312845+0000","last_fullsized":"2023-03-25T04:27:45.312845+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312882+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:27:45.312882+0000","last_peered":"2023-03-25T04:27:45.312882+0000","last_clean":"2023-03-25T04:27:45.312882+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:27:45.312882+0000","last_undegraded":"2023-03-25T04:27:45.312882+0000","last_fullsized":"2023-03-25T04:27:45.312882+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.311730+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:27:45.311730+0000","last_peered":"2023-03-25T04:27:45.311730+0000","last_clean":"2023-03-25T04:27:45.311730+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:27:45.311730+0000","last_undegraded":"2023-03-25T04:27:45.311730+0000","last_fullsized":"2023-03-25T04:27:45.311730+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315735+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:27:45.315735+0000","last_peered":"2023-03-25T04:27:45.315735+0000","last_clean":"2023-03-25T04:27:45.315735+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:27:45.315735+0000","last_undegraded":"2023-03-25T04:27:45.315735+0000","last_fullsized":"2023-03-25T04:27:45.315735+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312833+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:27:45.312833+0000","last_peered":"2023-03-25T04:27:45.312833+0000","last_clean":"2023-03-25T04:27:45.312833+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:27:45.312833+0000","last_undegraded":"2023-03-25T04:27:45.312833+0000","last_fullsized":"2023-03-25T04:27:45.312833+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117040,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:30:37 2023","interfaces":[{"interface":"back","average":{"1min":0.28799999999999998,"5min":0.28899999999999998,"15min":0.29499999999999998},"min":{"1min":0.20200000000000001,"5min":0.191,"15min":0.191},"max":{"1min":0.433,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.29799999999999999},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.28999999999999998,"15min":0.29799999999999999},"min":{"1min":0.17499999999999999,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":0.40300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.25700000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051630,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:30:32 2023","interfaces":[{"interface":"back","average":{"1min":0.25600000000000001,"5min":0.255,"15min":0.251},"min":{"1min":0.19400000000000001,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.35699999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"last":0.22800000000000001},{"interface":"front","average":{"1min":0.28699999999999998,"5min":0.27200000000000002,"15min":0.26600000000000001},"min":{"1min":0.20599999999999999,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.39400000000000002,"5min":0.42899999999999999,"15min":0.42899999999999999},"last":0.24099999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:31:08.343 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace: 2023-03-25T04:31:24.283+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:31:25.192+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:31:25.295+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)

Ignoring malparsed backtrace: 2023-03-25T04:31:40.257 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:31:40.271 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":212,"stamp":"2023-03-25T04:31:40.144732+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002208"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315468+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:27:45.315468+0000","last_peered":"2023-03-25T04:27:45.315468+0000","last_clean":"2023-03-25T04:27:45.315468+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:27:45.315468+0000","last_undegraded":"2023-03-25T04:27:45.315468+0000","last_fullsized":"2023-03-25T04:27:45.315468+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312845+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:27:45.312845+0000","last_peered":"2023-03-25T04:27:45.312845+0000","last_clean":"2023-03-25T04:27:45.312845+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:27:45.312845+0000","last_undegraded":"2023-03-25T04:27:45.312845+0000","last_fullsized":"2023-03-25T04:27:45.312845+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:23.308607+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:29:23.308607+0000","last_peered":"2023-03-25T04:29:23.308607+0000","last_clean":"2023-03-25T04:29:23.308607+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:29:23.308607+0000","last_undegraded":"2023-03-25T04:29:23.308607+0000","last_fullsized":"2023-03-25T04:29:23.308607+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":28,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:20.554710+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:29:20.554710+0000","last_peered":"2023-03-25T04:29:20.554710+0000","last_clean":"2023-03-25T04:29:20.554710+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:29:20.554710+0000","last_undegraded":"2023-03-25T04:29:20.554710+0000","last_fullsized":"2023-03-25T04:29:20.554710+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309216+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:29:28.309216+0000","last_peered":"2023-03-25T04:29:28.309216+0000","last_clean":"2023-03-25T04:29:28.309216+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:29:28.309216+0000","last_undegraded":"2023-03-25T04:29:28.309216+0000","last_fullsized":"2023-03-25T04:29:28.309216+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309233+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:29:28.309233+0000","last_peered":"2023-03-25T04:29:28.309233+0000","last_clean":"2023-03-25T04:29:28.309233+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:29:28.309233+0000","last_undegraded":"2023-03-25T04:29:28.309233+0000","last_fullsized":"2023-03-25T04:29:28.309233+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117046,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:30:37 2023","interfaces":[{"interface":"back","average":{"1min":0.28799999999999998,"5min":0.28899999999999998,"15min":0.29499999999999998},"min":{"1min":0.20200000000000001,"5min":0.191,"15min":0.191},"max":{"1min":0.433,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.216},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.28999999999999998,"15min":0.29799999999999999},"min":{"1min":0.17499999999999999,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":0.40300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.24299999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051637,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:31:35 2023","interfaces":[{"interface":"back","average":{"1min":0.26100000000000001,"5min":0.25700000000000001,"15min":0.251},"min":{"1min":0.19,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.35899999999999999,"5min":0.41599999999999998,"15min":0.41599999999999998},"last":0.221},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.27400000000000002,"15min":0.26700000000000002},"min":{"1min":0.20200000000000001,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.374,"5min":0.42899999999999999,"15min":0.42899999999999999},"last":0.24099999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:32:01.906+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:32:02.388+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:32:02.794 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:32:22.326 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":233,"stamp":"2023-03-25T04:32:22.153289+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002545"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:33.309748+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:29:33.309748+0000","last_peered":"2023-03-25T04:29:33.309748+0000","last_clean":"2023-03-25T04:29:33.309748+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:29:33.309748+0000","last_undegraded":"2023-03-25T04:29:33.309748+0000","last_fullsized":"2023-03-25T04:29:33.309748+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:33.309729+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:29:33.309729+0000","last_peered":"2023-03-25T04:29:33.309729+0000","last_clean":"2023-03-25T04:29:33.309729+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:29:33.309729+0000","last_undegraded":"2023-03-25T04:29:33.309729+0000","last_fullsized":"2023-03-25T04:29:33.309729+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:23.308607+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:29:23.308607+0000","last_peered":"2023-03-25T04:29:23.308607+0000","last_clean":"2023-03-25T04:29:23.308607+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:29:23.308607+0000","last_undegraded":"2023-03-25T04:29:23.308607+0000","last_fullsized":"2023-03-25T04:29:23.308607+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":28,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:20.554710+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:29:20.554710+0000","last_peered":"2023-03-25T04:29:20.554710+0000","last_clean":"2023-03-25T04:29:20.554710+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:29:20.554710+0000","last_undegraded":"2023-03-25T04:29:20.554710+0000","last_fullsized":"2023-03-25T04:29:20.554710+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309216+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:29:28.309216+0000","last_peered":"2023-03-25T04:29:28.309216+0000","last_clean":"2023-03-25T04:29:28.309216+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:29:28.309216+0000","last_undegraded":"2023-03-25T04:29:28.309216+0000","last_fullsized":"2023-03-25T04:29:28.309216+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309233+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:29:28.309233+0000","last_peered":"2023-03-25T04:29:28.309233+0000","last_clean":"2023-03-25T04:29:28.309233+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:29:28.309233+0000","last_undegraded":"2023-03-25T04:29:28.309233+0000","last_fullsized":"2023-03-25T04:29:28.309233+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117055,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:31:42 2023","interfaces":[{"interface":"back","average":{"1min":0.27500000000000002,"5min":0.28399999999999997,"15min":0.29399999999999998},"min":{"1min":0.19800000000000001,"5min":0.191,"15min":0.191},"max":{"1min":0.36899999999999999,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.32300000000000001},{"interface":"front","average":{"1min":0.27900000000000003,"5min":0.28499999999999998,"15min":0.29599999999999999},"min":{"1min":0.216,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":0.40799999999999997,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.27100000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051646,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:31:35 2023","interfaces":[{"interface":"back","average":{"1min":0.26100000000000001,"5min":0.25700000000000001,"15min":0.251},"min":{"1min":0.19,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.35899999999999999,"5min":0.41599999999999998,"15min":0.41599999999999998},"last":0.25600000000000001},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.27400000000000002,"15min":0.26700000000000002},"min":{"1min":0.20200000000000001,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.374,"5min":0.42899999999999999,"15min":0.42899999999999999},"last":0.29199999999999998}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:32:22.327 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean
, 2023-03-25T04:32:22.327 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}

Ignoring malparsed backtrace: 2023-03-25T04:32:39.836 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:32:39.849 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":241,"stamp":"2023-03-25T04:32:38.156312+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002346"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:33.309748+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:29:33.309748+0000","last_peered":"2023-03-25T04:29:33.309748+0000","last_clean":"2023-03-25T04:29:33.309748+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:29:33.309748+0000","last_undegraded":"2023-03-25T04:29:33.309748+0000","last_fullsized":"2023-03-25T04:29:33.309748+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:33.309729+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:29:33.309729+0000","last_peered":"2023-03-25T04:29:33.309729+0000","last_clean":"2023-03-25T04:29:33.309729+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:29:33.309729+0000","last_undegraded":"2023-03-25T04:29:33.309729+0000","last_fullsized":"2023-03-25T04:29:33.309729+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:23.308607+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:29:23.308607+0000","last_peered":"2023-03-25T04:29:23.308607+0000","last_clean":"2023-03-25T04:29:23.308607+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:29:23.308607+0000","last_undegraded":"2023-03-25T04:29:23.308607+0000","last_fullsized":"2023-03-25T04:29:23.308607+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":28,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:20.554710+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:29:20.554710+0000","last_peered":"2023-03-25T04:29:20.554710+0000","last_clean":"2023-03-25T04:29:20.554710+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:29:20.554710+0000","last_undegraded":"2023-03-25T04:29:20.554710+0000","last_fullsized":"2023-03-25T04:29:20.554710+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309216+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:29:28.309216+0000","last_peered":"2023-03-25T04:29:28.309216+0000","last_clean":"2023-03-25T04:29:28.309216+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:29:28.309216+0000","last_undegraded":"2023-03-25T04:29:28.309216+0000","last_fullsized":"2023-03-25T04:29:28.309216+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309233+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:29:28.309233+0000","last_peered":"2023-03-25T04:29:28.309233+0000","last_clean":"2023-03-25T04:29:28.309233+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:29:28.309233+0000","last_undegraded":"2023-03-25T04:29:28.309233+0000","last_fullsized":"2023-03-25T04:29:28.309233+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117058,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:31:42 2023","interfaces":[{"interface":"back","average":{"1min":0.27500000000000002,"5min":0.28399999999999997,"15min":0.29399999999999998},"min":{"1min":0.19800000000000001,"5min":0.191,"15min":0.191},"max":{"1min":0.36899999999999999,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.30599999999999999},{"interface":"front","average":{"1min":0.27900000000000003,"5min":0.28499999999999998,"15min":0.29599999999999999},"min":{"1min":0.216,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":0.40799999999999997,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.32500000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051649,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:31:35 2023","interfaces":[{"interface":"back","average":{"1min":0.26100000000000001,"5min":0.25700000000000001,"15min":0.251},"min":{"1min":0.19,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.35899999999999999,"5min":0.41599999999999998,"15min":0.41599999999999998},"last":0.24199999999999999},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.27400000000000002,"15min":0.26700000000000002},"min":{"1min":0.20200000000000001,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.374,"5min":0.42899999999999999,"15min":0.42899999999999999},"last":0.22600000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:32:54.944+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:32:55.469+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:32:55.927+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)

Ignoring malparsed backtrace: 2023-03-25T04:33:13.958 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean
, 2023-03-25T04:33:13.959 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}
, 2023-03-25T04:33:13.959 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json

Ignoring malparsed backtrace: 2023-03-25T04:33:31.199 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":267,"stamp":"2023-03-25T04:33:30.166660+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002269"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:33.309748+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:29:33.309748+0000","last_peered":"2023-03-25T04:29:33.309748+0000","last_clean":"2023-03-25T04:29:33.309748+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:29:33.309748+0000","last_undegraded":"2023-03-25T04:29:33.309748+0000","last_fullsized":"2023-03-25T04:29:33.309748+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:33.309729+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:29:33.309729+0000","last_peered":"2023-03-25T04:29:33.309729+0000","last_clean":"2023-03-25T04:29:33.309729+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:29:33.309729+0000","last_undegraded":"2023-03-25T04:29:33.309729+0000","last_fullsized":"2023-03-25T04:29:33.309729+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":28,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:31:23.320894+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:31:23.320894+0000","last_peered":"2023-03-25T04:31:23.320894+0000","last_clean":"2023-03-25T04:31:23.320894+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:31:23.320894+0000","last_undegraded":"2023-03-25T04:31:23.320894+0000","last_fullsized":"2023-03-25T04:31:23.320894+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":28,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:20.554710+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:29:20.554710+0000","last_peered":"2023-03-25T04:29:20.554710+0000","last_clean":"2023-03-25T04:29:20.554710+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:29:20.554710+0000","last_undegraded":"2023-03-25T04:29:20.554710+0000","last_fullsized":"2023-03-25T04:29:20.554710+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309216+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:29:28.309216+0000","last_peered":"2023-03-25T04:29:28.309216+0000","last_clean":"2023-03-25T04:29:28.309216+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:29:28.309216+0000","last_undegraded":"2023-03-25T04:29:28.309216+0000","last_fullsized":"2023-03-25T04:29:28.309216+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309233+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:29:28.309233+0000","last_peered":"2023-03-25T04:29:28.309233+0000","last_clean":"2023-03-25T04:29:28.309233+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:29:28.309233+0000","last_undegraded":"2023-03-25T04:29:28.309233+0000","last_fullsized":"2023-03-25T04:29:28.309233+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117068,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:32:46 2023","interfaces":[{"interface":"back","average":{"1min":0.27000000000000002,"5min":0.27900000000000003,"15min":0.29199999999999998},"min":{"1min":0.17899999999999999,"5min":0.17899999999999999,"15min":0.17899999999999999},"max":{"1min":0.35599999999999998,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.28799999999999998},{"interface":"front","average":{"1min":0.26800000000000002,"5min":0.27800000000000002,"15min":0.29399999999999998},"min":{"1min":0.19900000000000001,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":0.377,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.315}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051660,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:32:36 2023","interfaces":[{"interface":"back","average":{"1min":0.248,"5min":0.25700000000000001,"15min":0.251},"min":{"1min":0.17000000000000001,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"last":0.22700000000000001},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.27600000000000002,"15min":0.26800000000000002},"min":{"1min":0.192,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"last":0.20399999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:33:31.201 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:33:31.400+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)

Ignoring malparsed backtrace: 2023-03-25T04:33:47.531+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:33:47.951+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:33:48.505+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)

Ignoring malparsed backtrace: 2023-03-25T04:34:06.104 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json
, 2023-03-25T04:34:06.118+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:34:06.497 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:34:23.399 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:34:23.665+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:34:23.712 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:34:40.195+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:34:40.523+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:34:40.676 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace: 2023-03-25T04:35:03.381+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:35:04.139+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:35:04.391+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)

Ignoring malparsed backtrace: 2023-03-25T04:35:22.738 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean
, 2023-03-25T04:35:22.739 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}
, 2023-03-25T04:35:22.739 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json

Ignoring malparsed backtrace: 2023-03-25T04:35:41.393+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:35:42.352+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:35:42.376+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:35:57.384 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":340,"stamp":"2023-03-25T04:35:56.194992+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.001930"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":36,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:31:43.322894+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:31:43.322894+0000","last_peered":"2023-03-25T04:31:43.322894+0000","last_clean":"2023-03-25T04:31:43.322894+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:31:43.322894+0000","last_undegraded":"2023-03-25T04:31:43.322894+0000","last_fullsized":"2023-03-25T04:31:43.322894+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":28,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:31:43.322876+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:31:43.322876+0000","last_peered":"2023-03-25T04:31:43.322876+0000","last_clean":"2023-03-25T04:31:43.322876+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:31:43.322876+0000","last_undegraded":"2023-03-25T04:31:43.322876+0000","last_fullsized":"2023-03-25T04:31:43.322876+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:23.331998+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:33:23.331998+0000","last_peered":"2023-03-25T04:33:23.331998+0000","last_clean":"2023-03-25T04:33:23.331998+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:33:23.331998+0000","last_undegraded":"2023-03-25T04:33:23.331998+0000","last_fullsized":"2023-03-25T04:33:23.331998+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:30.576723+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:33:30.576723+0000","last_peered":"2023-03-25T04:33:30.576723+0000","last_clean":"2023-03-25T04:33:30.576723+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:33:30.576723+0000","last_undegraded":"2023-03-25T04:33:30.576723+0000","last_fullsized":"2023-03-25T04:33:30.576723+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":37,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:38.333447+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:33:38.333447+0000","last_peered":"2023-03-25T04:33:38.333447+0000","last_clean":"2023-03-25T04:33:38.333447+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:33:38.333447+0000","last_undegraded":"2023-03-25T04:33:38.333447+0000","last_fullsized":"2023-03-25T04:33:38.333447+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:38.333467+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:33:38.333467+0000","last_peered":"2023-03-25T04:33:38.333467+0000","last_clean":"2023-03-25T04:33:38.333467+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:33:38.333467+0000","last_undegraded":"2023-03-25T04:33:38.333467+0000","last_fullsized":"2023-03-25T04:33:38.333467+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117098,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:35:49 2023","interfaces":[{"interface":"back","average":{"1min":0.255,"5min":0.26000000000000001,"15min":0.28199999999999997},"min":{"1min":0.17899999999999999,"5min":0.16,"15min":0.16},"max":{"1min":0.36399999999999999,"5min":0.42599999999999999,"15min":0.45100000000000001},"last":0.26100000000000001},{"interface":"front","average":{"1min":0.26000000000000001,"5min":0.26700000000000002,"15min":0.28599999999999998},"min":{"1min":0.188,"5min":0.17999999999999999,"15min":0.17499999999999999},"max":{"1min":0.39100000000000001,"5min":0.45200000000000001,"15min":0.46300000000000002},"last":0.28399999999999997}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051689,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:35:42 2023","interfaces":[{"interface":"back","average":{"1min":0.29399999999999998,"5min":0.27200000000000002,"15min":0.25800000000000001},"min":{"1min":0.17299999999999999,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.41099999999999998,"5min":0.42699999999999999,"15min":0.42699999999999999},"last":0.20399999999999999},{"interface":"front","average":{"1min":0.30299999999999999,"5min":0.28399999999999997,"15min":0.27300000000000002},"min":{"1min":0.19700000000000001,"5min":0.192,"15min":0.16600000000000001},"max":{"1min":0.43099999999999999,"5min":0.45800000000000002,"15min":0.45800000000000002},"last":0.22}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:35:57.624+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)

Ignoring malparsed backtrace: 2023-03-25T04:36:17.226 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:36:17.239 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":350,"stamp":"2023-03-25T04:36:16.198866+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002148"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":37,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:53.335026+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:33:53.335026+0000","last_peered":"2023-03-25T04:33:53.335026+0000","last_clean":"2023-03-25T04:33:53.335026+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:33:53.335026+0000","last_undegraded":"2023-03-25T04:33:53.335026+0000","last_fullsized":"2023-03-25T04:33:53.335026+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:53.335009+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:33:53.335009+0000","last_peered":"2023-03-25T04:33:53.335009+0000","last_clean":"2023-03-25T04:33:53.335009+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:33:53.335009+0000","last_undegraded":"2023-03-25T04:33:53.335009+0000","last_fullsized":"2023-03-25T04:33:53.335009+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:23.331998+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:33:23.331998+0000","last_peered":"2023-03-25T04:33:23.331998+0000","last_clean":"2023-03-25T04:33:23.331998+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:33:23.331998+0000","last_undegraded":"2023-03-25T04:33:23.331998+0000","last_fullsized":"2023-03-25T04:33:23.331998+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:30.576723+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:33:30.576723+0000","last_peered":"2023-03-25T04:33:30.576723+0000","last_clean":"2023-03-25T04:33:30.576723+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:33:30.576723+0000","last_undegraded":"2023-03-25T04:33:30.576723+0000","last_fullsized":"2023-03-25T04:33:30.576723+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":37,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:38.333447+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:33:38.333447+0000","last_peered":"2023-03-25T04:33:38.333447+0000","last_clean":"2023-03-25T04:33:38.333447+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:33:38.333447+0000","last_undegraded":"2023-03-25T04:33:38.333447+0000","last_fullsized":"2023-03-25T04:33:38.333447+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:38.333467+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:33:38.333467+0000","last_peered":"2023-03-25T04:33:38.333467+0000","last_clean":"2023-03-25T04:33:38.333467+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:33:38.333467+0000","last_undegraded":"2023-03-25T04:33:38.333467+0000","last_fullsized":"2023-03-25T04:33:38.333467+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117102,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:35:49 2023","interfaces":[{"interface":"back","average":{"1min":0.255,"5min":0.26000000000000001,"15min":0.28199999999999997},"min":{"1min":0.17899999999999999,"5min":0.16,"15min":0.16},"max":{"1min":0.36399999999999999,"5min":0.42599999999999999,"15min":0.45100000000000001},"last":0.28399999999999997},{"interface":"front","average":{"1min":0.26000000000000001,"5min":0.26700000000000002,"15min":0.28599999999999998},"min":{"1min":0.188,"5min":0.17999999999999999,"15min":0.17499999999999999},"max":{"1min":0.39100000000000001,"5min":0.45200000000000001,"15min":0.46300000000000002},"last":0.30299999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051693,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:35:42 2023","interfaces":[{"interface":"back","average":{"1min":0.29399999999999998,"5min":0.27200000000000002,"15min":0.25800000000000001},"min":{"1min":0.17299999999999999,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.41099999999999998,"5min":0.42699999999999999,"15min":0.42699999999999999},"last":0.35099999999999998},{"interface":"front","average":{"1min":0.30299999999999999,"5min":0.28399999999999997,"15min":0.27300000000000002},"min":{"1min":0.19700000000000001,"5min":0.192,"15min":0.16600000000000001},"max":{"1min":0.43099999999999999,"5min":0.45800000000000002,"15min":0.45800000000000002},"last":0.32900000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:36:34.556+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:36:34.717+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:36:35.583+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)

Ignoring malparsed backtrace: 2023-03-25T04:36:52.680 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json
, 2023-03-25T04:36:52.701+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:36:53.067 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:37:09.963 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:37:10.273 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all

Ignoring malparsed backtrace: 2023-03-25T04:37:26.856+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:37:27.288 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:37:27.538+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:37:41.903 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":392,"stamp":"2023-03-25T04:37:40.213373+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002012"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":37,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:53.335026+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:33:53.335026+0000","last_peered":"2023-03-25T04:33:53.335026+0000","last_clean":"2023-03-25T04:33:53.335026+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:33:53.335026+0000","last_undegraded":"2023-03-25T04:33:53.335026+0000","last_fullsized":"2023-03-25T04:33:53.335026+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:53.335009+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:33:53.335009+0000","last_peered":"2023-03-25T04:33:53.335009+0000","last_clean":"2023-03-25T04:33:53.335009+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:33:53.335009+0000","last_undegraded":"2023-03-25T04:33:53.335009+0000","last_fullsized":"2023-03-25T04:33:53.335009+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:23.343585+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:35:23.343585+0000","last_peered":"2023-03-25T04:35:23.343585+0000","last_clean":"2023-03-25T04:35:23.343585+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:35:23.343585+0000","last_undegraded":"2023-03-25T04:35:23.343585+0000","last_fullsized":"2023-03-25T04:35:23.343585+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:30.576723+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:33:30.576723+0000","last_peered":"2023-03-25T04:33:30.576723+0000","last_clean":"2023-03-25T04:33:30.576723+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:33:30.576723+0000","last_undegraded":"2023-03-25T04:33:30.576723+0000","last_fullsized":"2023-03-25T04:33:30.576723+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":37,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:38.333447+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:33:38.333447+0000","last_peered":"2023-03-25T04:33:38.333447+0000","last_clean":"2023-03-25T04:33:38.333447+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:33:38.333447+0000","last_undegraded":"2023-03-25T04:33:38.333447+0000","last_fullsized":"2023-03-25T04:33:38.333447+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:38.333467+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:33:38.333467+0000","last_peered":"2023-03-25T04:33:38.333467+0000","last_clean":"2023-03-25T04:33:38.333467+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:33:38.333467+0000","last_undegraded":"2023-03-25T04:33:38.333467+0000","last_fullsized":"2023-03-25T04:33:38.333467+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117119,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:36:50 2023","interfaces":[{"interface":"back","average":{"1min":0.254,"5min":0.25600000000000001,"15min":0.27900000000000003},"min":{"1min":0.20499999999999999,"5min":0.16,"15min":0.16},"max":{"1min":0.371,"5min":0.42599999999999999,"15min":0.45100000000000001},"last":0.23000000000000001},{"interface":"front","average":{"1min":0.26800000000000002,"5min":0.26500000000000001,"15min":0.28399999999999997},"min":{"1min":0.19700000000000001,"5min":0.17999999999999999,"15min":0.17499999999999999},"max":{"1min":0.34699999999999998,"5min":0.45200000000000001,"15min":0.46300000000000002},"last":0.20899999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051710,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:36:44 2023","interfaces":[{"interface":"back","average":{"1min":0.28399999999999997,"5min":0.27700000000000002,"15min":0.26100000000000001},"min":{"1min":0.192,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.40899999999999997,"5min":0.42699999999999999,"15min":0.42699999999999999},"last":0.39700000000000002},{"interface":"front","average":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.27500000000000002},"min":{"1min":0.182,"5min":0.182,"15min":0.16600000000000001},"max":{"1min":0.439,"5min":0.45800000000000002,"15min":0.45800000000000002},"last":0.439}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:37:42.400+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)

Ignoring malparsed backtrace: 2023-03-25T04:38:04.766+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:38:04.922 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:38:05.246 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:38:19.605 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":411,"stamp":"2023-03-25T04:38:18.220966+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002368"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":38,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:36:03.347070+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:36:03.347070+0000","last_peered":"2023-03-25T04:36:03.347070+0000","last_clean":"2023-03-25T04:36:03.347070+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:36:03.347070+0000","last_undegraded":"2023-03-25T04:36:03.347070+0000","last_fullsized":"2023-03-25T04:36:03.347070+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:36:03.347025+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:36:03.347025+0000","last_peered":"2023-03-25T04:36:03.347025+0000","last_clean":"2023-03-25T04:36:03.347025+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:36:03.347025+0000","last_undegraded":"2023-03-25T04:36:03.347025+0000","last_fullsized":"2023-03-25T04:36:03.347025+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:23.343585+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:35:23.343585+0000","last_peered":"2023-03-25T04:35:23.343585+0000","last_clean":"2023-03-25T04:35:23.343585+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:35:23.343585+0000","last_undegraded":"2023-03-25T04:35:23.343585+0000","last_fullsized":"2023-03-25T04:35:23.343585+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:35.587025+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:35:35.587025+0000","last_peered":"2023-03-25T04:35:35.587025+0000","last_clean":"2023-03-25T04:35:35.587025+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:35:35.587025+0000","last_undegraded":"2023-03-25T04:35:35.587025+0000","last_fullsized":"2023-03-25T04:35:35.587025+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":38,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:43.345308+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:35:43.345308+0000","last_peered":"2023-03-25T04:35:43.345308+0000","last_clean":"2023-03-25T04:35:43.345308+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:35:43.345308+0000","last_undegraded":"2023-03-25T04:35:43.345308+0000","last_fullsized":"2023-03-25T04:35:43.345308+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:43.345324+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:35:43.345324+0000","last_peered":"2023-03-25T04:35:43.345324+0000","last_clean":"2023-03-25T04:35:43.345324+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:35:43.345324+0000","last_undegraded":"2023-03-25T04:35:43.345324+0000","last_fullsized":"2023-03-25T04:35:43.345324+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117127,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:37:53 2023","interfaces":[{"interface":"back","average":{"1min":0.25800000000000001,"5min":0.253,"15min":0.27700000000000002},"min":{"1min":0.184,"5min":0.16,"15min":0.16},"max":{"1min":0.44600000000000001,"5min":0.44600000000000001,"15min":0.45100000000000001},"last":0.35199999999999998},{"interface":"front","average":{"1min":0.26900000000000002,"5min":0.26500000000000001,"15min":0.28199999999999997},"min":{"1min":0.192,"5min":0.17999999999999999,"15min":0.17499999999999999},"max":{"1min":0.46400000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"last":0.33200000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051717,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:37:44 2023","interfaces":[{"interface":"back","average":{"1min":0.27300000000000002,"5min":0.28199999999999997,"15min":0.26200000000000001},"min":{"1min":0.17899999999999999,"5min":0.17299999999999999,"15min":0.17000000000000001},"max":{"1min":0.41799999999999998,"5min":0.42099999999999999,"15min":0.42699999999999999},"last":0.28699999999999998},{"interface":"front","average":{"1min":0.29399999999999998,"5min":0.29099999999999998,"15min":0.27700000000000002},"min":{"1min":0.19900000000000001,"5min":0.182,"15min":0.16600000000000001},"max":{"1min":0.441,"5min":0.441,"15min":0.45800000000000002},"last":0.26300000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:38:19.718+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:38:20.315+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)

Ignoring malparsed backtrace: 2023-03-25T04:38:44.283 INFO:teuthology.orchestra.run.smithi070.stdout:
, 2023-03-25T04:38:44.283 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":781,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":16,"modified":"2023-03-25T04:38:40.225321+0000","services":{}},"progress_events":{}}
, 2023-03-25T04:38:44.296 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now

Ignoring malparsed backtrace: 2023-03-25T04:39:02.477+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:39:02.578+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:39:03.524+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:39:20.359 INFO:teuthology.orchestra.run.smithi070.stdout:
, 2023-03-25T04:39:20.359 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":818,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":16,"modified":"2023-03-25T04:38:40.225321+0000","services":{}},"progress_events":{}}
, 2023-03-25T04:39:20.372 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now

Ignoring malparsed backtrace: 2023-03-25T04:39:37.527 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:39:37.541 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":450,"stamp":"2023-03-25T04:39:36.235332+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002086"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":38,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:36:03.347070+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:36:03.347070+0000","last_peered":"2023-03-25T04:36:03.347070+0000","last_clean":"2023-03-25T04:36:03.347070+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:36:03.347070+0000","last_undegraded":"2023-03-25T04:36:03.347070+0000","last_fullsized":"2023-03-25T04:36:03.347070+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:36:03.347025+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:36:03.347025+0000","last_peered":"2023-03-25T04:36:03.347025+0000","last_clean":"2023-03-25T04:36:03.347025+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:36:03.347025+0000","last_undegraded":"2023-03-25T04:36:03.347025+0000","last_fullsized":"2023-03-25T04:36:03.347025+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:23.353778+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:37:23.353778+0000","last_peered":"2023-03-25T04:37:23.353778+0000","last_clean":"2023-03-25T04:37:23.353778+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:37:23.353778+0000","last_undegraded":"2023-03-25T04:37:23.353778+0000","last_fullsized":"2023-03-25T04:37:23.353778+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:35.587025+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:35:35.587025+0000","last_peered":"2023-03-25T04:35:35.587025+0000","last_clean":"2023-03-25T04:35:35.587025+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:35:35.587025+0000","last_undegraded":"2023-03-25T04:35:35.587025+0000","last_fullsized":"2023-03-25T04:35:35.587025+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":38,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:43.345308+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:35:43.345308+0000","last_peered":"2023-03-25T04:35:43.345308+0000","last_clean":"2023-03-25T04:35:43.345308+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:35:43.345308+0000","last_undegraded":"2023-03-25T04:35:43.345308+0000","last_fullsized":"2023-03-25T04:35:43.345308+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:43.345324+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:35:43.345324+0000","last_peered":"2023-03-25T04:35:43.345324+0000","last_clean":"2023-03-25T04:35:43.345324+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:35:43.345324+0000","last_undegraded":"2023-03-25T04:35:43.345324+0000","last_fullsized":"2023-03-25T04:35:43.345324+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117143,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:38:54 2023","interfaces":[{"interface":"back","average":{"1min":0.25800000000000001,"5min":0.25700000000000001,"15min":0.27400000000000002},"min":{"1min":0.192,"5min":0.17399999999999999,"15min":0.16},"max":{"1min":0.39400000000000002,"5min":0.44600000000000001,"15min":0.45100000000000001},"last":0.26500000000000001},{"interface":"front","average":{"1min":0.25900000000000001,"5min":0.26700000000000002,"15min":0.27900000000000003},"min":{"1min":0.19500000000000001,"5min":0.188,"15min":0.17499999999999999},"max":{"1min":0.41499999999999998,"5min":0.46400000000000002,"15min":0.46400000000000002},"last":0.29099999999999998}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051733,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:38:45 2023","interfaces":[{"interface":"back","average":{"1min":0.27300000000000002,"5min":0.28100000000000003,"15min":0.26400000000000001},"min":{"1min":0.19400000000000001,"5min":0.17299999999999999,"15min":0.17000000000000001},"max":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"last":0.27300000000000002},{"interface":"front","average":{"1min":0.28799999999999998,"5min":0.29099999999999998,"15min":0.27800000000000002},"min":{"1min":0.19800000000000001,"5min":0.182,"15min":0.16600000000000001},"max":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"last":0.308}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:39:54.416+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:39:54.455 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:39:54.780 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:40:09.288+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:40:09.389+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:40:10.267+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)

Ignoring malparsed backtrace: 2023-03-25T04:40:28.984 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":476,"stamp":"2023-03-25T04:40:28.245911+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002693"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":39,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:38:13.358968+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:38:13.358968+0000","last_peered":"2023-03-25T04:38:13.358968+0000","last_clean":"2023-03-25T04:38:13.358968+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:38:13.358968+0000","last_undegraded":"2023-03-25T04:38:13.358968+0000","last_fullsized":"2023-03-25T04:38:13.358968+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:38:13.358922+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:38:13.358922+0000","last_peered":"2023-03-25T04:38:13.358922+0000","last_clean":"2023-03-25T04:38:13.358922+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:38:13.358922+0000","last_undegraded":"2023-03-25T04:38:13.358922+0000","last_fullsized":"2023-03-25T04:38:13.358922+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:23.353778+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:37:23.353778+0000","last_peered":"2023-03-25T04:37:23.353778+0000","last_clean":"2023-03-25T04:37:23.353778+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:37:23.353778+0000","last_undegraded":"2023-03-25T04:37:23.353778+0000","last_fullsized":"2023-03-25T04:37:23.353778+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:40.598579+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:37:40.598579+0000","last_peered":"2023-03-25T04:37:40.598579+0000","last_clean":"2023-03-25T04:37:40.598579+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:37:40.598579+0000","last_undegraded":"2023-03-25T04:37:40.598579+0000","last_fullsized":"2023-03-25T04:37:40.598579+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":39,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:48.356344+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:37:48.356344+0000","last_peered":"2023-03-25T04:37:48.356344+0000","last_clean":"2023-03-25T04:37:48.356344+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:37:48.356344+0000","last_undegraded":"2023-03-25T04:37:48.356344+0000","last_fullsized":"2023-03-25T04:37:48.356344+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:48.356365+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:37:48.356365+0000","last_peered":"2023-03-25T04:37:48.356365+0000","last_clean":"2023-03-25T04:37:48.356365+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:37:48.356365+0000","last_undegraded":"2023-03-25T04:37:48.356365+0000","last_fullsized":"2023-03-25T04:37:48.356365+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117153,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:39:54 2023","interfaces":[{"interface":"back","average":{"1min":0.25900000000000001,"5min":0.25600000000000001,"15min":0.27100000000000002},"min":{"1min":0.188,"5min":0.17899999999999999,"15min":0.16},"max":{"1min":0.41999999999999998,"5min":0.44600000000000001,"15min":0.45100000000000001},"last":0.255},{"interface":"front","average":{"1min":0.27500000000000002,"5min":0.26600000000000001,"15min":0.27700000000000002},"min":{"1min":0.17699999999999999,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.45600000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"last":0.26800000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051743,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:39:47 2023","interfaces":[{"interface":"back","average":{"1min":0.26300000000000001,"5min":0.27700000000000002,"15min":0.26500000000000001},"min":{"1min":0.187,"5min":0.17299999999999999,"15min":0.17000000000000001},"max":{"1min":0.433,"5min":0.433,"15min":0.433},"last":0.36599999999999999},{"interface":"front","average":{"1min":0.27100000000000002,"5min":0.28799999999999998,"15min":0.27900000000000003},"min":{"1min":0.20100000000000001,"5min":0.182,"15min":0.16600000000000001},"max":{"1min":0.48599999999999999,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.34999999999999998}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:40:28.986 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean
, 2023-03-25T04:40:28.986 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}

Ignoring malparsed backtrace: 2023-03-25T04:40:47.229+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:40:47.503+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:40:48.251+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)

Ignoring malparsed backtrace: 2023-03-25T04:41:03.267 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:41:03.578 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all

Ignoring malparsed backtrace: 2023-03-25T04:41:21.347 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":502,"stamp":"2023-03-25T04:41:20.256204+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002342"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":39,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:38:13.358968+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:38:13.358968+0000","last_peered":"2023-03-25T04:38:13.358968+0000","last_clean":"2023-03-25T04:38:13.358968+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:38:13.358968+0000","last_undegraded":"2023-03-25T04:38:13.358968+0000","last_fullsized":"2023-03-25T04:38:13.358968+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:38:13.358922+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:38:13.358922+0000","last_peered":"2023-03-25T04:38:13.358922+0000","last_clean":"2023-03-25T04:38:13.358922+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:38:13.358922+0000","last_undegraded":"2023-03-25T04:38:13.358922+0000","last_fullsized":"2023-03-25T04:38:13.358922+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:23.353778+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:37:23.353778+0000","last_peered":"2023-03-25T04:37:23.353778+0000","last_clean":"2023-03-25T04:37:23.353778+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:37:23.353778+0000","last_undegraded":"2023-03-25T04:37:23.353778+0000","last_fullsized":"2023-03-25T04:37:23.353778+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:40.598579+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:37:40.598579+0000","last_peered":"2023-03-25T04:37:40.598579+0000","last_clean":"2023-03-25T04:37:40.598579+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:37:40.598579+0000","last_undegraded":"2023-03-25T04:37:40.598579+0000","last_fullsized":"2023-03-25T04:37:40.598579+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":39,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:48.356344+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:37:48.356344+0000","last_peered":"2023-03-25T04:37:48.356344+0000","last_clean":"2023-03-25T04:37:48.356344+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:37:48.356344+0000","last_undegraded":"2023-03-25T04:37:48.356344+0000","last_fullsized":"2023-03-25T04:37:48.356344+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:48.356365+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:37:48.356365+0000","last_peered":"2023-03-25T04:37:48.356365+0000","last_clean":"2023-03-25T04:37:48.356365+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:37:48.356365+0000","last_undegraded":"2023-03-25T04:37:48.356365+0000","last_fullsized":"2023-03-25T04:37:48.356365+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117163,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:40:56 2023","interfaces":[{"interface":"back","average":{"1min":0.28799999999999998,"5min":0.26300000000000001,"15min":0.27100000000000002},"min":{"1min":0.188,"5min":0.184,"15min":0.16},"max":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.17499999999999999},{"interface":"front","average":{"1min":0.29599999999999999,"5min":0.27300000000000002,"15min":0.27700000000000002},"min":{"1min":0.20599999999999999,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.43099999999999999,"5min":0.46400000000000002,"15min":0.46400000000000002},"last":0.20699999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051754,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:40:52 2023","interfaces":[{"interface":"back","average":{"1min":0.28599999999999998,"5min":0.27500000000000002,"15min":0.26700000000000002},"min":{"1min":0.17899999999999999,"5min":0.17899999999999999,"15min":0.17000000000000001},"max":{"1min":0.44,"5min":0.44,"15min":0.44},"last":0.23699999999999999},{"interface":"front","average":{"1min":0.29799999999999999,"5min":0.28699999999999998,"15min":0.28100000000000003},"min":{"1min":0.20300000000000001,"5min":0.182,"15min":0.16600000000000001},"max":{"1min":0.47599999999999998,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.28000000000000003}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:41:21.349 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean
, 2023-03-25T04:41:21.349 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}

Ignoring malparsed backtrace: 2023-03-25T04:41:39.867+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:41:40.383+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:41:40.905+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:41:58.219 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json
, 2023-03-25T04:41:58.319+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:41:58.644 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:42:18.193 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":530,"stamp":"2023-03-25T04:42:16.266659+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002356"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":39,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:38:13.358968+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:38:13.358968+0000","last_peered":"2023-03-25T04:38:13.358968+0000","last_clean":"2023-03-25T04:38:13.358968+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:38:13.358968+0000","last_undegraded":"2023-03-25T04:38:13.358968+0000","last_fullsized":"2023-03-25T04:38:13.358968+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:38:13.358922+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:38:13.358922+0000","last_peered":"2023-03-25T04:38:13.358922+0000","last_clean":"2023-03-25T04:38:13.358922+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:38:13.358922+0000","last_undegraded":"2023-03-25T04:38:13.358922+0000","last_fullsized":"2023-03-25T04:38:13.358922+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:23.364593+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:39:23.364593+0000","last_peered":"2023-03-25T04:39:23.364593+0000","last_clean":"2023-03-25T04:39:23.364593+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:39:23.364593+0000","last_undegraded":"2023-03-25T04:39:23.364593+0000","last_fullsized":"2023-03-25T04:39:23.364593+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:45.608475+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:39:45.608475+0000","last_peered":"2023-03-25T04:39:45.608475+0000","last_clean":"2023-03-25T04:39:45.608475+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:39:45.608475+0000","last_undegraded":"2023-03-25T04:39:45.608475+0000","last_fullsized":"2023-03-25T04:39:45.608475+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":40,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:53.367398+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:39:53.367398+0000","last_peered":"2023-03-25T04:39:53.367398+0000","last_clean":"2023-03-25T04:39:53.367398+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:39:53.367398+0000","last_undegraded":"2023-03-25T04:39:53.367398+0000","last_fullsized":"2023-03-25T04:39:53.367398+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:53.367424+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:39:53.367424+0000","last_peered":"2023-03-25T04:39:53.367424+0000","last_clean":"2023-03-25T04:39:53.367424+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:39:53.367424+0000","last_undegraded":"2023-03-25T04:39:53.367424+0000","last_fullsized":"2023-03-25T04:39:53.367424+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117175,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:41:58 2023","interfaces":[{"interface":"back","average":{"1min":0.25800000000000001,"5min":0.26400000000000001,"15min":0.26800000000000002},"min":{"1min":0.17499999999999999,"5min":0.17499999999999999,"15min":0.16},"max":{"1min":0.39100000000000001,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.22900000000000001},{"interface":"front","average":{"1min":0.26300000000000001,"5min":0.27200000000000002,"15min":0.27400000000000002},"min":{"1min":0.20599999999999999,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.35699999999999998,"5min":0.46400000000000002,"15min":0.46400000000000002},"last":0.26400000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051765,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:41:52 2023","interfaces":[{"interface":"back","average":{"1min":0.26500000000000001,"5min":0.27200000000000002,"15min":0.26800000000000002},"min":{"1min":0.17000000000000001,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.39200000000000002,"5min":0.44,"15min":0.44},"last":0.23899999999999999},{"interface":"front","average":{"1min":0.28100000000000003,"5min":0.28599999999999998,"15min":0.28199999999999997},"min":{"1min":0.19400000000000001,"5min":0.19400000000000001,"15min":0.16600000000000001},"max":{"1min":0.41399999999999998,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.254}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:42:18.297+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:42:18.958+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:42:35.512 INFO:teuthology.orchestra.run.smithi070.stdout:
, 2023-03-25T04:42:35.512 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":1013,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":17,"modified":"2023-03-25T04:41:54.262744+0000","services":{}},"progress_events":{}}
, 2023-03-25T04:42:35.525 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:42:52.741 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":548,"stamp":"2023-03-25T04:42:52.273601+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002221"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":40,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:40:23.370056+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:40:23.370056+0000","last_peered":"2023-03-25T04:40:23.370056+0000","last_clean":"2023-03-25T04:40:23.370056+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:40:23.370056+0000","last_undegraded":"2023-03-25T04:40:23.370056+0000","last_fullsized":"2023-03-25T04:40:23.370056+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:40:23.370039+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:40:23.370039+0000","last_peered":"2023-03-25T04:40:23.370039+0000","last_clean":"2023-03-25T04:40:23.370039+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:40:23.370039+0000","last_undegraded":"2023-03-25T04:40:23.370039+0000","last_fullsized":"2023-03-25T04:40:23.370039+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:23.364593+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:39:23.364593+0000","last_peered":"2023-03-25T04:39:23.364593+0000","last_clean":"2023-03-25T04:39:23.364593+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:39:23.364593+0000","last_undegraded":"2023-03-25T04:39:23.364593+0000","last_fullsized":"2023-03-25T04:39:23.364593+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:45.608475+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:39:45.608475+0000","last_peered":"2023-03-25T04:39:45.608475+0000","last_clean":"2023-03-25T04:39:45.608475+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:39:45.608475+0000","last_undegraded":"2023-03-25T04:39:45.608475+0000","last_fullsized":"2023-03-25T04:39:45.608475+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":40,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:53.367398+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:39:53.367398+0000","last_peered":"2023-03-25T04:39:53.367398+0000","last_clean":"2023-03-25T04:39:53.367398+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:39:53.367398+0000","last_undegraded":"2023-03-25T04:39:53.367398+0000","last_fullsized":"2023-03-25T04:39:53.367398+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:53.367424+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:39:53.367424+0000","last_peered":"2023-03-25T04:39:53.367424+0000","last_clean":"2023-03-25T04:39:53.367424+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:39:53.367424+0000","last_undegraded":"2023-03-25T04:39:53.367424+0000","last_fullsized":"2023-03-25T04:39:53.367424+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117182,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:41:58 2023","interfaces":[{"interface":"back","average":{"1min":0.25800000000000001,"5min":0.26400000000000001,"15min":0.26800000000000002},"min":{"1min":0.17499999999999999,"5min":0.17499999999999999,"15min":0.16},"max":{"1min":0.39100000000000001,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.19600000000000001},{"interface":"front","average":{"1min":0.26300000000000001,"5min":0.27200000000000002,"15min":0.27400000000000002},"min":{"1min":0.20599999999999999,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.35699999999999998,"5min":0.46400000000000002,"15min":0.46400000000000002},"last":0.219}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051773,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:41:52 2023","interfaces":[{"interface":"back","average":{"1min":0.26500000000000001,"5min":0.27200000000000002,"15min":0.26800000000000002},"min":{"1min":0.17000000000000001,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.39200000000000002,"5min":0.44,"15min":0.44},"last":0.19},{"interface":"front","average":{"1min":0.28100000000000003,"5min":0.28599999999999998,"15min":0.28199999999999997},"min":{"1min":0.19400000000000001,"5min":0.19400000000000001,"15min":0.16600000000000001},"max":{"1min":0.41399999999999998,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.17599999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:42:52.742 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean

Ignoring malparsed backtrace: 2023-03-25T04:43:09.973 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:43:09.986 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":556,"stamp":"2023-03-25T04:43:08.276556+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002359"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":40,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:40:23.370056+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:40:23.370056+0000","last_peered":"2023-03-25T04:40:23.370056+0000","last_clean":"2023-03-25T04:40:23.370056+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:40:23.370056+0000","last_undegraded":"2023-03-25T04:40:23.370056+0000","last_fullsized":"2023-03-25T04:40:23.370056+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:40:23.370039+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:40:23.370039+0000","last_peered":"2023-03-25T04:40:23.370039+0000","last_clean":"2023-03-25T04:40:23.370039+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:40:23.370039+0000","last_undegraded":"2023-03-25T04:40:23.370039+0000","last_fullsized":"2023-03-25T04:40:23.370039+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:23.364593+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:39:23.364593+0000","last_peered":"2023-03-25T04:39:23.364593+0000","last_clean":"2023-03-25T04:39:23.364593+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:39:23.364593+0000","last_undegraded":"2023-03-25T04:39:23.364593+0000","last_fullsized":"2023-03-25T04:39:23.364593+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:45.608475+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:39:45.608475+0000","last_peered":"2023-03-25T04:39:45.608475+0000","last_clean":"2023-03-25T04:39:45.608475+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:39:45.608475+0000","last_undegraded":"2023-03-25T04:39:45.608475+0000","last_fullsized":"2023-03-25T04:39:45.608475+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":40,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:53.367398+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:39:53.367398+0000","last_peered":"2023-03-25T04:39:53.367398+0000","last_clean":"2023-03-25T04:39:53.367398+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:39:53.367398+0000","last_undegraded":"2023-03-25T04:39:53.367398+0000","last_fullsized":"2023-03-25T04:39:53.367398+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:53.367424+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:39:53.367424+0000","last_peered":"2023-03-25T04:39:53.367424+0000","last_clean":"2023-03-25T04:39:53.367424+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:39:53.367424+0000","last_undegraded":"2023-03-25T04:39:53.367424+0000","last_fullsized":"2023-03-25T04:39:53.367424+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117185,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:43:00 2023","interfaces":[{"interface":"back","average":{"1min":0.27400000000000002,"5min":0.26700000000000002,"15min":0.26600000000000001},"min":{"1min":0.19600000000000001,"5min":0.17499999999999999,"15min":0.16},"max":{"1min":0.42999999999999999,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.27800000000000002},{"interface":"front","average":{"1min":0.28199999999999997,"5min":0.27500000000000002,"15min":0.27300000000000002},"min":{"1min":0.185,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.47199999999999998,"5min":0.47199999999999998,"15min":0.47199999999999998},"last":0.247}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051776,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:42:54 2023","interfaces":[{"interface":"back","average":{"1min":0.26300000000000001,"5min":0.27000000000000002,"15min":0.26900000000000002},"min":{"1min":0.17399999999999999,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.438,"5min":0.44,"15min":0.44},"last":0.20100000000000001},{"interface":"front","average":{"1min":0.27400000000000002,"5min":0.28199999999999997,"15min":0.28299999999999997},"min":{"1min":0.17599999999999999,"5min":0.17599999999999999,"15min":0.16600000000000001},"max":{"1min":0.41599999999999998,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.23499999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:43:25.042+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:43:25.411+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:43:26.013+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:43:47.937+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:43:48.534+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:43:48.889+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:44:09.318 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:44:09.331 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":586,"stamp":"2023-03-25T04:44:08.288933+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002697"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":40,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:40:23.370056+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:40:23.370056+0000","last_peered":"2023-03-25T04:40:23.370056+0000","last_clean":"2023-03-25T04:40:23.370056+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:40:23.370056+0000","last_undegraded":"2023-03-25T04:40:23.370056+0000","last_fullsized":"2023-03-25T04:40:23.370056+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:40:23.370039+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:40:23.370039+0000","last_peered":"2023-03-25T04:40:23.370039+0000","last_clean":"2023-03-25T04:40:23.370039+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:40:23.370039+0000","last_undegraded":"2023-03-25T04:40:23.370039+0000","last_fullsized":"2023-03-25T04:40:23.370039+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:23.375106+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:41:23.375106+0000","last_peered":"2023-03-25T04:41:23.375106+0000","last_clean":"2023-03-25T04:41:23.375106+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:41:23.375106+0000","last_undegraded":"2023-03-25T04:41:23.375106+0000","last_fullsized":"2023-03-25T04:41:23.375106+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:50.618930+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:41:50.618930+0000","last_peered":"2023-03-25T04:41:50.618930+0000","last_clean":"2023-03-25T04:41:50.618930+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:41:50.618930+0000","last_undegraded":"2023-03-25T04:41:50.618930+0000","last_fullsized":"2023-03-25T04:41:50.618930+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":41,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:58.377821+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:41:58.377821+0000","last_peered":"2023-03-25T04:41:58.377821+0000","last_clean":"2023-03-25T04:41:58.377821+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:41:58.377821+0000","last_undegraded":"2023-03-25T04:41:58.377821+0000","last_fullsized":"2023-03-25T04:41:58.377821+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:58.377836+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:41:58.377836+0000","last_peered":"2023-03-25T04:41:58.377836+0000","last_clean":"2023-03-25T04:41:58.377836+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:41:58.377836+0000","last_undegraded":"2023-03-25T04:41:58.377836+0000","last_fullsized":"2023-03-25T04:41:58.377836+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117197,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:44:01 2023","interfaces":[{"interface":"back","average":{"1min":0.27200000000000002,"5min":0.27000000000000002,"15min":0.26400000000000001},"min":{"1min":0.20399999999999999,"5min":0.17499999999999999,"15min":0.16},"max":{"1min":0.38400000000000001,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.245},{"interface":"front","average":{"1min":0.27600000000000002,"5min":0.27800000000000002,"15min":0.27100000000000002},"min":{"1min":0.23100000000000001,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.40400000000000003,"5min":0.47199999999999998,"15min":0.47199999999999998},"last":0.27500000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051788,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:43:55 2023","interfaces":[{"interface":"back","average":{"1min":0.26100000000000001,"5min":0.26700000000000002,"15min":0.27000000000000002},"min":{"1min":0.17799999999999999,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.374,"5min":0.44,"15min":0.44},"last":0.30299999999999999},{"interface":"front","average":{"1min":0.28299999999999997,"5min":0.28100000000000003,"15min":0.28399999999999997},"min":{"1min":0.20899999999999999,"5min":0.17599999999999999,"15min":0.16600000000000001},"max":{"1min":0.40100000000000002,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.27800000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:44:28.923 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":1126,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":19,"modified":"2023-03-25T04:43:20.278959+0000","services":{}},"progress_events":{}}
, 2023-03-25T04:44:28.936 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now
, 2023-03-25T04:44:28.937 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace: 2023-03-25T04:44:48.786 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:44:48.800 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":606,"stamp":"2023-03-25T04:44:48.296805+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002182"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":41,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:42:34.382196+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:42:34.382196+0000","last_peered":"2023-03-25T04:42:34.382196+0000","last_clean":"2023-03-25T04:42:34.382196+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:42:34.382196+0000","last_undegraded":"2023-03-25T04:42:34.382196+0000","last_fullsized":"2023-03-25T04:42:34.382196+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:42:29.381744+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:42:29.381744+0000","last_peered":"2023-03-25T04:42:29.381744+0000","last_clean":"2023-03-25T04:42:29.381744+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:42:29.381744+0000","last_undegraded":"2023-03-25T04:42:29.381744+0000","last_fullsized":"2023-03-25T04:42:29.381744+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:23.375106+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:41:23.375106+0000","last_peered":"2023-03-25T04:41:23.375106+0000","last_clean":"2023-03-25T04:41:23.375106+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:41:23.375106+0000","last_undegraded":"2023-03-25T04:41:23.375106+0000","last_fullsized":"2023-03-25T04:41:23.375106+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:50.618930+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:41:50.618930+0000","last_peered":"2023-03-25T04:41:50.618930+0000","last_clean":"2023-03-25T04:41:50.618930+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:41:50.618930+0000","last_undegraded":"2023-03-25T04:41:50.618930+0000","last_fullsized":"2023-03-25T04:41:50.618930+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":41,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:58.377821+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:41:58.377821+0000","last_peered":"2023-03-25T04:41:58.377821+0000","last_clean":"2023-03-25T04:41:58.377821+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:41:58.377821+0000","last_undegraded":"2023-03-25T04:41:58.377821+0000","last_fullsized":"2023-03-25T04:41:58.377821+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:58.377836+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:41:58.377836+0000","last_peered":"2023-03-25T04:41:58.377836+0000","last_clean":"2023-03-25T04:41:58.377836+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:41:58.377836+0000","last_undegraded":"2023-03-25T04:41:58.377836+0000","last_fullsized":"2023-03-25T04:41:58.377836+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117206,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:44:01 2023","interfaces":[{"interface":"back","average":{"1min":0.27200000000000002,"5min":0.27000000000000002,"15min":0.26400000000000001},"min":{"1min":0.20399999999999999,"5min":0.17499999999999999,"15min":0.16},"max":{"1min":0.38400000000000001,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.216},{"interface":"front","average":{"1min":0.27600000000000002,"5min":0.27800000000000002,"15min":0.27100000000000002},"min":{"1min":0.23100000000000001,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.40400000000000003,"5min":0.47199999999999998,"15min":0.47199999999999998},"last":0.23999999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051796,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:43:55 2023","interfaces":[{"interface":"back","average":{"1min":0.26100000000000001,"5min":0.26700000000000002,"15min":0.27000000000000002},"min":{"1min":0.17799999999999999,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.374,"5min":0.44,"15min":0.44},"last":0.25600000000000001},{"interface":"front","average":{"1min":0.28299999999999997,"5min":0.28100000000000003,"15min":0.28399999999999997},"min":{"1min":0.20899999999999999,"5min":0.17599999999999999,"15min":0.16600000000000001},"max":{"1min":0.40100000000000002,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.28000000000000003}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:45:05.762 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:45:06.073 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all

Ignoring malparsed backtrace: 2023-03-25T04:45:25.658 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":624,"stamp":"2023-03-25T04:45:24.304294+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002366"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":41,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:42:34.382196+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:42:34.382196+0000","last_peered":"2023-03-25T04:42:34.382196+0000","last_clean":"2023-03-25T04:42:34.382196+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:42:34.382196+0000","last_undegraded":"2023-03-25T04:42:34.382196+0000","last_fullsized":"2023-03-25T04:42:34.382196+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:42:29.381744+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:42:29.381744+0000","last_peered":"2023-03-25T04:42:29.381744+0000","last_clean":"2023-03-25T04:42:29.381744+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:42:29.381744+0000","last_undegraded":"2023-03-25T04:42:29.381744+0000","last_fullsized":"2023-03-25T04:42:29.381744+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:23.375106+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:41:23.375106+0000","last_peered":"2023-03-25T04:41:23.375106+0000","last_clean":"2023-03-25T04:41:23.375106+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:41:23.375106+0000","last_undegraded":"2023-03-25T04:41:23.375106+0000","last_fullsized":"2023-03-25T04:41:23.375106+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:50.618930+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:41:50.618930+0000","last_peered":"2023-03-25T04:41:50.618930+0000","last_clean":"2023-03-25T04:41:50.618930+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:41:50.618930+0000","last_undegraded":"2023-03-25T04:41:50.618930+0000","last_fullsized":"2023-03-25T04:41:50.618930+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":41,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:58.377821+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:41:58.377821+0000","last_peered":"2023-03-25T04:41:58.377821+0000","last_clean":"2023-03-25T04:41:58.377821+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:41:58.377821+0000","last_undegraded":"2023-03-25T04:41:58.377821+0000","last_fullsized":"2023-03-25T04:41:58.377821+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:58.377836+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:41:58.377836+0000","last_peered":"2023-03-25T04:41:58.377836+0000","last_clean":"2023-03-25T04:41:58.377836+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:41:58.377836+0000","last_undegraded":"2023-03-25T04:41:58.377836+0000","last_fullsized":"2023-03-25T04:41:58.377836+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117213,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:45:01 2023","interfaces":[{"interface":"back","average":{"1min":0.251,"5min":0.26800000000000002,"15min":0.26400000000000001},"min":{"1min":0.183,"5min":0.17499999999999999,"15min":0.16},"max":{"1min":0.41499999999999998,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.26900000000000002},{"interface":"front","average":{"1min":0.28000000000000003,"5min":0.27900000000000003,"15min":0.27100000000000002},"min":{"1min":0.20999999999999999,"5min":0.185,"15min":0.17499999999999999},"max":{"1min":0.44400000000000001,"5min":0.47199999999999998,"15min":0.47199999999999998},"last":0.27900000000000003}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051803,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:44:55 2023","interfaces":[{"interface":"back","average":{"1min":0.252,"5min":0.26500000000000001,"15min":0.26900000000000002},"min":{"1min":0.183,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.39700000000000002,"5min":0.44,"15min":0.44},"last":0.20699999999999999},{"interface":"front","average":{"1min":0.27000000000000002,"5min":0.28100000000000003,"15min":0.28299999999999997},"min":{"1min":0.20300000000000001,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":0.42099999999999999,"5min":0.47599999999999998,"15min":0.48599999999999999},"last":0.23599999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:45:25.659 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:45:25.969 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:45:42.735+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:45:42.744+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:45:42.897 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace: 2023-03-25T04:46:00.183 INFO:teuthology.orchestra.run.smithi070.stdout:
, 2023-03-25T04:46:00.183 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":1217,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":20,"modified":"2023-03-25T04:45:42.307700+0000","services":{"mgr":{"daemons":{"summary":"","x":{"start_epoch":0,"start_stamp":"0.000000","gid":0,"addr":"(unrecognized address family 0)/0","metadata":{},"task_status":{}}}}}},"progress_events":{}}
, 2023-03-25T04:46:00.197 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now

Ignoring malparsed backtrace: 2023-03-25T04:46:19.910+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:46:20.039 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all

Ignoring malparsed backtrace: 2023-03-25T04:46:39.611 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}
, 2023-03-25T04:46:39.611 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json
, 2023-03-25T04:46:39.613+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:46:58.485+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:46:58.764+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:46:59.519+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:47:14.132 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:47:14.146 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":678,"stamp":"2023-03-25T04:47:12.325044+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002571"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":42,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:44.393426+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:44:44.393426+0000","last_peered":"2023-03-25T04:44:44.393426+0000","last_clean":"2023-03-25T04:44:44.393426+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:44:44.393426+0000","last_undegraded":"2023-03-25T04:44:44.393426+0000","last_fullsized":"2023-03-25T04:44:44.393426+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:39.393044+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:44:39.393044+0000","last_peered":"2023-03-25T04:44:39.393044+0000","last_clean":"2023-03-25T04:44:39.393044+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:44:39.393044+0000","last_undegraded":"2023-03-25T04:44:39.393044+0000","last_fullsized":"2023-03-25T04:44:39.393044+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:43:24.386177+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:43:24.386177+0000","last_peered":"2023-03-25T04:43:24.386177+0000","last_clean":"2023-03-25T04:43:24.386177+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:43:24.386177+0000","last_undegraded":"2023-03-25T04:43:24.386177+0000","last_fullsized":"2023-03-25T04:43:24.386177+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:43:55.629798+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:43:55.629798+0000","last_peered":"2023-03-25T04:43:55.629798+0000","last_clean":"2023-03-25T04:43:55.629798+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:43:55.629798+0000","last_undegraded":"2023-03-25T04:43:55.629798+0000","last_fullsized":"2023-03-25T04:43:55.629798+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":42,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:04.389904+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:44:04.389904+0000","last_peered":"2023-03-25T04:44:04.389904+0000","last_clean":"2023-03-25T04:44:04.389904+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:44:04.389904+0000","last_undegraded":"2023-03-25T04:44:04.389904+0000","last_fullsized":"2023-03-25T04:44:04.389904+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:04.389923+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:44:04.389923+0000","last_peered":"2023-03-25T04:44:04.389923+0000","last_clean":"2023-03-25T04:44:04.389923+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:44:04.389923+0000","last_undegraded":"2023-03-25T04:44:04.389923+0000","last_fullsized":"2023-03-25T04:44:04.389923+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117235,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:47:05 2023","interfaces":[{"interface":"back","average":{"1min":0.27800000000000002,"5min":0.27000000000000002,"15min":0.26300000000000001},"min":{"1min":0.20100000000000001,"5min":0.16900000000000001,"15min":0.16},"max":{"1min":0.39200000000000002,"5min":0.42999999999999999,"15min":0.45600000000000002},"last":0.379},{"interface":"front","average":{"1min":0.28399999999999997,"5min":0.28100000000000003,"15min":0.27200000000000002},"min":{"1min":0.17799999999999999,"5min":0.17799999999999999,"15min":0.17699999999999999},"max":{"1min":0.42599999999999999,"5min":0.47199999999999998,"15min":0.47199999999999998},"last":0.39800000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051825,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:47:01 2023","interfaces":[{"interface":"back","average":{"1min":0.27300000000000002,"5min":0.26100000000000001,"15min":0.27000000000000002},"min":{"1min":0.186,"5min":0.17399999999999999,"15min":0.17000000000000001},"max":{"1min":0.40400000000000003,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.23899999999999999},{"interface":"front","average":{"1min":0.28699999999999998,"5min":0.27500000000000002,"15min":0.28299999999999997},"min":{"1min":0.186,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.48599999999999999},"last":0.19600000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:47:31.198 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json
, 2023-03-25T04:47:31.411+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:47:31.789 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:47:49.509 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":696,"stamp":"2023-03-25T04:47:48.332093+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002697"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":42,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:44.393426+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:44:44.393426+0000","last_peered":"2023-03-25T04:44:44.393426+0000","last_clean":"2023-03-25T04:44:44.393426+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:44:44.393426+0000","last_undegraded":"2023-03-25T04:44:44.393426+0000","last_fullsized":"2023-03-25T04:44:44.393426+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:39.393044+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:44:39.393044+0000","last_peered":"2023-03-25T04:44:39.393044+0000","last_clean":"2023-03-25T04:44:39.393044+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:44:39.393044+0000","last_undegraded":"2023-03-25T04:44:39.393044+0000","last_fullsized":"2023-03-25T04:44:39.393044+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:45:24.396717+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:45:24.396717+0000","last_peered":"2023-03-25T04:45:24.396717+0000","last_clean":"2023-03-25T04:45:24.396717+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:45:24.396717+0000","last_undegraded":"2023-03-25T04:45:24.396717+0000","last_fullsized":"2023-03-25T04:45:24.396717+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:43:55.629798+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:43:55.629798+0000","last_peered":"2023-03-25T04:43:55.629798+0000","last_clean":"2023-03-25T04:43:55.629798+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:43:55.629798+0000","last_undegraded":"2023-03-25T04:43:55.629798+0000","last_fullsized":"2023-03-25T04:43:55.629798+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":42,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:04.389904+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:44:04.389904+0000","last_peered":"2023-03-25T04:44:04.389904+0000","last_clean":"2023-03-25T04:44:04.389904+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:44:04.389904+0000","last_undegraded":"2023-03-25T04:44:04.389904+0000","last_fullsized":"2023-03-25T04:44:04.389904+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:04.389923+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:44:04.389923+0000","last_peered":"2023-03-25T04:44:04.389923+0000","last_clean":"2023-03-25T04:44:04.389923+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:44:04.389923+0000","last_undegraded":"2023-03-25T04:44:04.389923+0000","last_fullsized":"2023-03-25T04:44:04.389923+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117242,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:47:05 2023","interfaces":[{"interface":"back","average":{"1min":0.27800000000000002,"5min":0.27000000000000002,"15min":0.26300000000000001},"min":{"1min":0.20100000000000001,"5min":0.16900000000000001,"15min":0.16},"max":{"1min":0.39200000000000002,"5min":0.42999999999999999,"15min":0.45600000000000002},"last":0.40500000000000003},{"interface":"front","average":{"1min":0.28399999999999997,"5min":0.28100000000000003,"15min":0.27200000000000002},"min":{"1min":0.17799999999999999,"5min":0.17799999999999999,"15min":0.17699999999999999},"max":{"1min":0.42599999999999999,"5min":0.47199999999999998,"15min":0.47199999999999998},"last":0.42399999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051834,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:47:01 2023","interfaces":[{"interface":"back","average":{"1min":0.27300000000000002,"5min":0.26100000000000001,"15min":0.27000000000000002},"min":{"1min":0.186,"5min":0.17399999999999999,"15min":0.17000000000000001},"max":{"1min":0.40400000000000003,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.25600000000000001},{"interface":"front","average":{"1min":0.28699999999999998,"5min":0.27500000000000002,"15min":0.28299999999999997},"min":{"1min":0.186,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.48599999999999999},"last":0.22800000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:47:49.510 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace:     ctx.managers[config['cluster']].wait_for_clean()
,   File "/home/teuthworker/src/github.com_ceph_ceph-c_511015c23eef0684a64171b710e432448df2b7c7/qa/tasks/ceph_manager.py", line 2747, in wait_for_clean
,     assert time.time() - start < timeout, \

Ignoring malparsed backtrace:  ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
,  1: /lib64/libpthread.so.0(+0x12cf0) [0x7f1453c2fcf0]
,  2: __pthread_mutex_lock()

Ignoring malparsed backtrace: 2023-03-25T04:28:05.172 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":142,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":5,"modified":"2023-03-25T04:27:12.092098+0000","services":{}},"progress_events":{}}
, 2023-03-25T04:28:05.185 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now
, 2023-03-25T04:28:05.186 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace: 2023-03-25T04:28:28.360 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":116,"stamp":"2023-03-25T04:28:28.108007+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002578"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.314661+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:27:45.314661+0000","last_peered":"2023-03-25T04:27:45.314661+0000","last_clean":"2023-03-25T04:27:45.314661+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:27:45.314661+0000","last_undegraded":"2023-03-25T04:27:45.314661+0000","last_fullsized":"2023-03-25T04:27:45.314661+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":25,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.310745+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:27:44.310745+0000","last_peered":"2023-03-25T04:27:44.310745+0000","last_clean":"2023-03-25T04:27:44.310745+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:27:44.310745+0000","last_undegraded":"2023-03-25T04:27:44.310745+0000","last_fullsized":"2023-03-25T04:27:44.310745+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":25,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.310650+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:27:44.310650+0000","last_peered":"2023-03-25T04:27:44.310650+0000","last_clean":"2023-03-25T04:27:44.310650+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:27:44.310650+0000","last_undegraded":"2023-03-25T04:27:44.310650+0000","last_fullsized":"2023-03-25T04:27:44.310650+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":26,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.312962+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:27:44.312962+0000","last_peered":"2023-03-25T04:27:44.312962+0000","last_clean":"2023-03-25T04:27:44.312962+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:27:44.312962+0000","last_undegraded":"2023-03-25T04:27:44.312962+0000","last_fullsized":"2023-03-25T04:27:44.312962+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315064+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:27:45.315064+0000","last_peered":"2023-03-25T04:27:45.315064+0000","last_clean":"2023-03-25T04:27:45.315064+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:27:45.315064+0000","last_undegraded":"2023-03-25T04:27:45.315064+0000","last_fullsized":"2023-03-25T04:27:45.315064+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":25,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.310707+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:27:44.310707+0000","last_peered":"2023-03-25T04:27:44.310707+0000","last_clean":"2023-03-25T04:27:44.310707+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:27:44.310707+0000","last_undegraded":"2023-03-25T04:27:44.310707+0000","last_fullsized":"2023-03-25T04:27:44.310707+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117007,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.375}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051597,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.186}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:28:28.362 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean
, 2023-03-25T04:28:28.362 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:28:45.655 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":124,"stamp":"2023-03-25T04:28:44.111265+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002492"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.314661+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:27:45.314661+0000","last_peered":"2023-03-25T04:27:45.314661+0000","last_clean":"2023-03-25T04:27:45.314661+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:27:45.314661+0000","last_undegraded":"2023-03-25T04:27:45.314661+0000","last_fullsized":"2023-03-25T04:27:45.314661+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":25,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.310745+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:27:44.310745+0000","last_peered":"2023-03-25T04:27:44.310745+0000","last_clean":"2023-03-25T04:27:44.310745+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:27:44.310745+0000","last_undegraded":"2023-03-25T04:27:44.310745+0000","last_fullsized":"2023-03-25T04:27:44.310745+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":25,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.310650+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:27:44.310650+0000","last_peered":"2023-03-25T04:27:44.310650+0000","last_clean":"2023-03-25T04:27:44.310650+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:27:44.310650+0000","last_undegraded":"2023-03-25T04:27:44.310650+0000","last_fullsized":"2023-03-25T04:27:44.310650+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":26,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.312962+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:27:44.312962+0000","last_peered":"2023-03-25T04:27:44.312962+0000","last_clean":"2023-03-25T04:27:44.312962+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:27:44.312962+0000","last_undegraded":"2023-03-25T04:27:44.312962+0000","last_fullsized":"2023-03-25T04:27:44.312962+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315064+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:27:45.315064+0000","last_peered":"2023-03-25T04:27:45.315064+0000","last_clean":"2023-03-25T04:27:45.315064+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:27:45.315064+0000","last_undegraded":"2023-03-25T04:27:45.315064+0000","last_fullsized":"2023-03-25T04:27:45.315064+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":25,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.310707+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:27:44.310707+0000","last_peered":"2023-03-25T04:27:44.310707+0000","last_clean":"2023-03-25T04:27:44.310707+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:27:44.310707+0000","last_undegraded":"2023-03-25T04:27:44.310707+0000","last_fullsized":"2023-03-25T04:27:44.310707+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117011,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:28:31 2023","interfaces":[{"interface":"back","average":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.36899999999999999},{"interface":"front","average":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"min":{"1min":0.19,"5min":0.19,"15min":0.19},"max":{"1min":0.46300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.39800000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051601,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:28:24 2023","interfaces":[{"interface":"back","average":{"1min":0.249,"5min":0.249,"15min":0.249},"min":{"1min":0.17699999999999999,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"last":0.254},{"interface":"front","average":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"min":{"1min":0.20200000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"last":0.312}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:28:45.656 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace: 2023-03-25T04:29:05.171 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}
, 2023-03-25T04:29:05.171 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json
, 2023-03-25T04:29:05.561 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:29:22.465 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:29:22.773 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all

Ignoring malparsed backtrace: 2023-03-25T04:29:42.405 INFO:teuthology.orchestra.run.smithi070.stdout:
, 2023-03-25T04:29:42.406 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":240,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":9,"modified":"2023-03-25T04:29:32.120358+0000","services":{"mon":{"daemons":{"summary":"","a":{"start_epoch":0,"start_stamp":"0.000000","gid":0,"addr":"(unrecognized address family 0)/0","metadata":{},"task_status":{}}}}}},"progress_events":{}}
, 2023-03-25T04:29:42.419 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:29:59.627 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":161,"stamp":"2023-03-25T04:29:58.125186+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002398"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315468+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:27:45.315468+0000","last_peered":"2023-03-25T04:27:45.315468+0000","last_clean":"2023-03-25T04:27:45.315468+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:27:45.315468+0000","last_undegraded":"2023-03-25T04:27:45.315468+0000","last_fullsized":"2023-03-25T04:27:45.315468+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312845+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:27:45.312845+0000","last_peered":"2023-03-25T04:27:45.312845+0000","last_clean":"2023-03-25T04:27:45.312845+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:27:45.312845+0000","last_undegraded":"2023-03-25T04:27:45.312845+0000","last_fullsized":"2023-03-25T04:27:45.312845+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312882+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:27:45.312882+0000","last_peered":"2023-03-25T04:27:45.312882+0000","last_clean":"2023-03-25T04:27:45.312882+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:27:45.312882+0000","last_undegraded":"2023-03-25T04:27:45.312882+0000","last_fullsized":"2023-03-25T04:27:45.312882+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.311730+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:27:45.311730+0000","last_peered":"2023-03-25T04:27:45.311730+0000","last_clean":"2023-03-25T04:27:45.311730+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:27:45.311730+0000","last_undegraded":"2023-03-25T04:27:45.311730+0000","last_fullsized":"2023-03-25T04:27:45.311730+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315735+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:27:45.315735+0000","last_peered":"2023-03-25T04:27:45.315735+0000","last_clean":"2023-03-25T04:27:45.315735+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:27:45.315735+0000","last_undegraded":"2023-03-25T04:27:45.315735+0000","last_fullsized":"2023-03-25T04:27:45.315735+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312833+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:27:45.312833+0000","last_peered":"2023-03-25T04:27:45.312833+0000","last_clean":"2023-03-25T04:27:45.312833+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:27:45.312833+0000","last_undegraded":"2023-03-25T04:27:45.312833+0000","last_fullsized":"2023-03-25T04:27:45.312833+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117026,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:29:33 2023","interfaces":[{"interface":"back","average":{"1min":0.26300000000000001,"5min":0.29099999999999998,"15min":0.29599999999999999},"min":{"1min":0.191,"5min":0.191,"15min":0.191},"max":{"1min":0.371,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.20499999999999999},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.29599999999999999,"15min":0.29999999999999999},"min":{"1min":0.21099999999999999,"5min":0.19,"15min":0.19},"max":{"1min":0.39800000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.19500000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051616,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:29:29 2023","interfaces":[{"interface":"back","average":{"1min":0.27200000000000002,"5min":0.253,"15min":0.25},"min":{"1min":0.183,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"last":0.26400000000000001},{"interface":"front","average":{"1min":0.28499999999999998,"5min":0.26800000000000002,"15min":0.26500000000000001},"min":{"1min":0.16600000000000001,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"last":0.29899999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:29:59.629 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean

Ignoring malparsed backtrace: 2023-03-25T04:30:16.889 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:30:16.903 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":170,"stamp":"2023-03-25T04:30:16.128427+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002222"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315468+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:27:45.315468+0000","last_peered":"2023-03-25T04:27:45.315468+0000","last_clean":"2023-03-25T04:27:45.315468+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:27:45.315468+0000","last_undegraded":"2023-03-25T04:27:45.315468+0000","last_fullsized":"2023-03-25T04:27:45.315468+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312845+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:27:45.312845+0000","last_peered":"2023-03-25T04:27:45.312845+0000","last_clean":"2023-03-25T04:27:45.312845+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:27:45.312845+0000","last_undegraded":"2023-03-25T04:27:45.312845+0000","last_fullsized":"2023-03-25T04:27:45.312845+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312882+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:27:45.312882+0000","last_peered":"2023-03-25T04:27:45.312882+0000","last_clean":"2023-03-25T04:27:45.312882+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:27:45.312882+0000","last_undegraded":"2023-03-25T04:27:45.312882+0000","last_fullsized":"2023-03-25T04:27:45.312882+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.311730+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:27:45.311730+0000","last_peered":"2023-03-25T04:27:45.311730+0000","last_clean":"2023-03-25T04:27:45.311730+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:27:45.311730+0000","last_undegraded":"2023-03-25T04:27:45.311730+0000","last_fullsized":"2023-03-25T04:27:45.311730+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315735+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:27:45.315735+0000","last_peered":"2023-03-25T04:27:45.315735+0000","last_clean":"2023-03-25T04:27:45.315735+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:27:45.315735+0000","last_undegraded":"2023-03-25T04:27:45.315735+0000","last_fullsized":"2023-03-25T04:27:45.315735+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312833+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:27:45.312833+0000","last_peered":"2023-03-25T04:27:45.312833+0000","last_clean":"2023-03-25T04:27:45.312833+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:27:45.312833+0000","last_undegraded":"2023-03-25T04:27:45.312833+0000","last_fullsized":"2023-03-25T04:27:45.312833+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117030,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:29:33 2023","interfaces":[{"interface":"back","average":{"1min":0.26300000000000001,"5min":0.29099999999999998,"15min":0.29599999999999999},"min":{"1min":0.191,"5min":0.191,"15min":0.191},"max":{"1min":0.371,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.245},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.29599999999999999,"15min":0.29999999999999999},"min":{"1min":0.21099999999999999,"5min":0.19,"15min":0.19},"max":{"1min":0.39800000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.20899999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051620,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:29:29 2023","interfaces":[{"interface":"back","average":{"1min":0.27200000000000002,"5min":0.253,"15min":0.25},"min":{"1min":0.183,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"last":0.19400000000000001},{"interface":"front","average":{"1min":0.28499999999999998,"5min":0.26800000000000002,"15min":0.26500000000000001},"min":{"1min":0.16600000000000001,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"last":0.23599999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:30:31.984+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:30:32.201+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:30:32.981+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)

Ignoring malparsed backtrace: 2023-03-25T04:30:48.449 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now
, 2023-03-25T04:30:48.450 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:30:48.760 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:31:08.342 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":196,"stamp":"2023-03-25T04:31:08.139018+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002431"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315468+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:27:45.315468+0000","last_peered":"2023-03-25T04:27:45.315468+0000","last_clean":"2023-03-25T04:27:45.315468+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:27:45.315468+0000","last_undegraded":"2023-03-25T04:27:45.315468+0000","last_fullsized":"2023-03-25T04:27:45.315468+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312845+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:27:45.312845+0000","last_peered":"2023-03-25T04:27:45.312845+0000","last_clean":"2023-03-25T04:27:45.312845+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:27:45.312845+0000","last_undegraded":"2023-03-25T04:27:45.312845+0000","last_fullsized":"2023-03-25T04:27:45.312845+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312882+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:27:45.312882+0000","last_peered":"2023-03-25T04:27:45.312882+0000","last_clean":"2023-03-25T04:27:45.312882+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:27:45.312882+0000","last_undegraded":"2023-03-25T04:27:45.312882+0000","last_fullsized":"2023-03-25T04:27:45.312882+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.311730+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:27:45.311730+0000","last_peered":"2023-03-25T04:27:45.311730+0000","last_clean":"2023-03-25T04:27:45.311730+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:27:45.311730+0000","last_undegraded":"2023-03-25T04:27:45.311730+0000","last_fullsized":"2023-03-25T04:27:45.311730+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315735+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:27:45.315735+0000","last_peered":"2023-03-25T04:27:45.315735+0000","last_clean":"2023-03-25T04:27:45.315735+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:27:45.315735+0000","last_undegraded":"2023-03-25T04:27:45.315735+0000","last_fullsized":"2023-03-25T04:27:45.315735+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312833+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:27:45.312833+0000","last_peered":"2023-03-25T04:27:45.312833+0000","last_clean":"2023-03-25T04:27:45.312833+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:27:45.312833+0000","last_undegraded":"2023-03-25T04:27:45.312833+0000","last_fullsized":"2023-03-25T04:27:45.312833+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117040,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:30:37 2023","interfaces":[{"interface":"back","average":{"1min":0.28799999999999998,"5min":0.28899999999999998,"15min":0.29499999999999998},"min":{"1min":0.20200000000000001,"5min":0.191,"15min":0.191},"max":{"1min":0.433,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.29799999999999999},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.28999999999999998,"15min":0.29799999999999999},"min":{"1min":0.17499999999999999,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":0.40300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.25700000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051630,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:30:32 2023","interfaces":[{"interface":"back","average":{"1min":0.25600000000000001,"5min":0.255,"15min":0.251},"min":{"1min":0.19400000000000001,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.35699999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"last":0.22800000000000001},{"interface":"front","average":{"1min":0.28699999999999998,"5min":0.27200000000000002,"15min":0.26600000000000001},"min":{"1min":0.20599999999999999,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.39400000000000002,"5min":0.42899999999999999,"15min":0.42899999999999999},"last":0.24099999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:31:08.343 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace: 2023-03-25T04:31:24.283+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:31:25.192+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:31:25.295+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)

Ignoring malparsed backtrace: 2023-03-25T04:31:40.257 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:31:40.271 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":212,"stamp":"2023-03-25T04:31:40.144732+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002208"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315468+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:27:45.315468+0000","last_peered":"2023-03-25T04:27:45.315468+0000","last_clean":"2023-03-25T04:27:45.315468+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:27:45.315468+0000","last_undegraded":"2023-03-25T04:27:45.315468+0000","last_fullsized":"2023-03-25T04:27:45.315468+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312845+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:27:45.312845+0000","last_peered":"2023-03-25T04:27:45.312845+0000","last_clean":"2023-03-25T04:27:45.312845+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:27:45.312845+0000","last_undegraded":"2023-03-25T04:27:45.312845+0000","last_fullsized":"2023-03-25T04:27:45.312845+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:23.308607+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:29:23.308607+0000","last_peered":"2023-03-25T04:29:23.308607+0000","last_clean":"2023-03-25T04:29:23.308607+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:29:23.308607+0000","last_undegraded":"2023-03-25T04:29:23.308607+0000","last_fullsized":"2023-03-25T04:29:23.308607+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":28,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:20.554710+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:29:20.554710+0000","last_peered":"2023-03-25T04:29:20.554710+0000","last_clean":"2023-03-25T04:29:20.554710+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:29:20.554710+0000","last_undegraded":"2023-03-25T04:29:20.554710+0000","last_fullsized":"2023-03-25T04:29:20.554710+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309216+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:29:28.309216+0000","last_peered":"2023-03-25T04:29:28.309216+0000","last_clean":"2023-03-25T04:29:28.309216+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:29:28.309216+0000","last_undegraded":"2023-03-25T04:29:28.309216+0000","last_fullsized":"2023-03-25T04:29:28.309216+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309233+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:29:28.309233+0000","last_peered":"2023-03-25T04:29:28.309233+0000","last_clean":"2023-03-25T04:29:28.309233+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:29:28.309233+0000","last_undegraded":"2023-03-25T04:29:28.309233+0000","last_fullsized":"2023-03-25T04:29:28.309233+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117046,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:30:37 2023","interfaces":[{"interface":"back","average":{"1min":0.28799999999999998,"5min":0.28899999999999998,"15min":0.29499999999999998},"min":{"1min":0.20200000000000001,"5min":0.191,"15min":0.191},"max":{"1min":0.433,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.216},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.28999999999999998,"15min":0.29799999999999999},"min":{"1min":0.17499999999999999,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":0.40300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.24299999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051637,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:31:35 2023","interfaces":[{"interface":"back","average":{"1min":0.26100000000000001,"5min":0.25700000000000001,"15min":0.251},"min":{"1min":0.19,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.35899999999999999,"5min":0.41599999999999998,"15min":0.41599999999999998},"last":0.221},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.27400000000000002,"15min":0.26700000000000002},"min":{"1min":0.20200000000000001,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.374,"5min":0.42899999999999999,"15min":0.42899999999999999},"last":0.24099999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:32:01.906+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:32:02.388+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:32:02.794 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:32:22.326 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":233,"stamp":"2023-03-25T04:32:22.153289+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002545"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:33.309748+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:29:33.309748+0000","last_peered":"2023-03-25T04:29:33.309748+0000","last_clean":"2023-03-25T04:29:33.309748+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:29:33.309748+0000","last_undegraded":"2023-03-25T04:29:33.309748+0000","last_fullsized":"2023-03-25T04:29:33.309748+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:33.309729+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:29:33.309729+0000","last_peered":"2023-03-25T04:29:33.309729+0000","last_clean":"2023-03-25T04:29:33.309729+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:29:33.309729+0000","last_undegraded":"2023-03-25T04:29:33.309729+0000","last_fullsized":"2023-03-25T04:29:33.309729+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:23.308607+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:29:23.308607+0000","last_peered":"2023-03-25T04:29:23.308607+0000","last_clean":"2023-03-25T04:29:23.308607+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:29:23.308607+0000","last_undegraded":"2023-03-25T04:29:23.308607+0000","last_fullsized":"2023-03-25T04:29:23.308607+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":28,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:20.554710+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:29:20.554710+0000","last_peered":"2023-03-25T04:29:20.554710+0000","last_clean":"2023-03-25T04:29:20.554710+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:29:20.554710+0000","last_undegraded":"2023-03-25T04:29:20.554710+0000","last_fullsized":"2023-03-25T04:29:20.554710+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309216+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:29:28.309216+0000","last_peered":"2023-03-25T04:29:28.309216+0000","last_clean":"2023-03-25T04:29:28.309216+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:29:28.309216+0000","last_undegraded":"2023-03-25T04:29:28.309216+0000","last_fullsized":"2023-03-25T04:29:28.309216+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309233+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:29:28.309233+0000","last_peered":"2023-03-25T04:29:28.309233+0000","last_clean":"2023-03-25T04:29:28.309233+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:29:28.309233+0000","last_undegraded":"2023-03-25T04:29:28.309233+0000","last_fullsized":"2023-03-25T04:29:28.309233+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117055,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:31:42 2023","interfaces":[{"interface":"back","average":{"1min":0.27500000000000002,"5min":0.28399999999999997,"15min":0.29399999999999998},"min":{"1min":0.19800000000000001,"5min":0.191,"15min":0.191},"max":{"1min":0.36899999999999999,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.32300000000000001},{"interface":"front","average":{"1min":0.27900000000000003,"5min":0.28499999999999998,"15min":0.29599999999999999},"min":{"1min":0.216,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":0.40799999999999997,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.27100000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051646,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:31:35 2023","interfaces":[{"interface":"back","average":{"1min":0.26100000000000001,"5min":0.25700000000000001,"15min":0.251},"min":{"1min":0.19,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.35899999999999999,"5min":0.41599999999999998,"15min":0.41599999999999998},"last":0.25600000000000001},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.27400000000000002,"15min":0.26700000000000002},"min":{"1min":0.20200000000000001,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.374,"5min":0.42899999999999999,"15min":0.42899999999999999},"last":0.29199999999999998}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:32:22.327 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean
, 2023-03-25T04:32:22.327 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}

Ignoring malparsed backtrace: 2023-03-25T04:32:39.836 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:32:39.849 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":241,"stamp":"2023-03-25T04:32:38.156312+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002346"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:33.309748+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:29:33.309748+0000","last_peered":"2023-03-25T04:29:33.309748+0000","last_clean":"2023-03-25T04:29:33.309748+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:29:33.309748+0000","last_undegraded":"2023-03-25T04:29:33.309748+0000","last_fullsized":"2023-03-25T04:29:33.309748+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:33.309729+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:29:33.309729+0000","last_peered":"2023-03-25T04:29:33.309729+0000","last_clean":"2023-03-25T04:29:33.309729+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:29:33.309729+0000","last_undegraded":"2023-03-25T04:29:33.309729+0000","last_fullsized":"2023-03-25T04:29:33.309729+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:23.308607+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:29:23.308607+0000","last_peered":"2023-03-25T04:29:23.308607+0000","last_clean":"2023-03-25T04:29:23.308607+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:29:23.308607+0000","last_undegraded":"2023-03-25T04:29:23.308607+0000","last_fullsized":"2023-03-25T04:29:23.308607+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":28,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:20.554710+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:29:20.554710+0000","last_peered":"2023-03-25T04:29:20.554710+0000","last_clean":"2023-03-25T04:29:20.554710+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:29:20.554710+0000","last_undegraded":"2023-03-25T04:29:20.554710+0000","last_fullsized":"2023-03-25T04:29:20.554710+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309216+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:29:28.309216+0000","last_peered":"2023-03-25T04:29:28.309216+0000","last_clean":"2023-03-25T04:29:28.309216+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:29:28.309216+0000","last_undegraded":"2023-03-25T04:29:28.309216+0000","last_fullsized":"2023-03-25T04:29:28.309216+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309233+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:29:28.309233+0000","last_peered":"2023-03-25T04:29:28.309233+0000","last_clean":"2023-03-25T04:29:28.309233+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:29:28.309233+0000","last_undegraded":"2023-03-25T04:29:28.309233+0000","last_fullsized":"2023-03-25T04:29:28.309233+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117058,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:31:42 2023","interfaces":[{"interface":"back","average":{"1min":0.27500000000000002,"5min":0.28399999999999997,"15min":0.29399999999999998},"min":{"1min":0.19800000000000001,"5min":0.191,"15min":0.191},"max":{"1min":0.36899999999999999,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.30599999999999999},{"interface":"front","average":{"1min":0.27900000000000003,"5min":0.28499999999999998,"15min":0.29599999999999999},"min":{"1min":0.216,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":0.40799999999999997,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.32500000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051649,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:31:35 2023","interfaces":[{"interface":"back","average":{"1min":0.26100000000000001,"5min":0.25700000000000001,"15min":0.251},"min":{"1min":0.19,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.35899999999999999,"5min":0.41599999999999998,"15min":0.41599999999999998},"last":0.24199999999999999},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.27400000000000002,"15min":0.26700000000000002},"min":{"1min":0.20200000000000001,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.374,"5min":0.42899999999999999,"15min":0.42899999999999999},"last":0.22600000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:32:54.944+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:32:55.469+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:32:55.927+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)

Ignoring malparsed backtrace: 2023-03-25T04:33:13.958 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean
, 2023-03-25T04:33:13.959 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}
, 2023-03-25T04:33:13.959 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json

Ignoring malparsed backtrace: 2023-03-25T04:33:31.199 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":267,"stamp":"2023-03-25T04:33:30.166660+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002269"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:33.309748+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:29:33.309748+0000","last_peered":"2023-03-25T04:29:33.309748+0000","last_clean":"2023-03-25T04:29:33.309748+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:29:33.309748+0000","last_undegraded":"2023-03-25T04:29:33.309748+0000","last_fullsized":"2023-03-25T04:29:33.309748+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:33.309729+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:29:33.309729+0000","last_peered":"2023-03-25T04:29:33.309729+0000","last_clean":"2023-03-25T04:29:33.309729+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:29:33.309729+0000","last_undegraded":"2023-03-25T04:29:33.309729+0000","last_fullsized":"2023-03-25T04:29:33.309729+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":28,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:31:23.320894+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:31:23.320894+0000","last_peered":"2023-03-25T04:31:23.320894+0000","last_clean":"2023-03-25T04:31:23.320894+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:31:23.320894+0000","last_undegraded":"2023-03-25T04:31:23.320894+0000","last_fullsized":"2023-03-25T04:31:23.320894+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":28,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:20.554710+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:29:20.554710+0000","last_peered":"2023-03-25T04:29:20.554710+0000","last_clean":"2023-03-25T04:29:20.554710+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:29:20.554710+0000","last_undegraded":"2023-03-25T04:29:20.554710+0000","last_fullsized":"2023-03-25T04:29:20.554710+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309216+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:29:28.309216+0000","last_peered":"2023-03-25T04:29:28.309216+0000","last_clean":"2023-03-25T04:29:28.309216+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:29:28.309216+0000","last_undegraded":"2023-03-25T04:29:28.309216+0000","last_fullsized":"2023-03-25T04:29:28.309216+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309233+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:29:28.309233+0000","last_peered":"2023-03-25T04:29:28.309233+0000","last_clean":"2023-03-25T04:29:28.309233+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:29:28.309233+0000","last_undegraded":"2023-03-25T04:29:28.309233+0000","last_fullsized":"2023-03-25T04:29:28.309233+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117068,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:32:46 2023","interfaces":[{"interface":"back","average":{"1min":0.27000000000000002,"5min":0.27900000000000003,"15min":0.29199999999999998},"min":{"1min":0.17899999999999999,"5min":0.17899999999999999,"15min":0.17899999999999999},"max":{"1min":0.35599999999999998,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.28799999999999998},{"interface":"front","average":{"1min":0.26800000000000002,"5min":0.27800000000000002,"15min":0.29399999999999998},"min":{"1min":0.19900000000000001,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":0.377,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.315}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051660,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:32:36 2023","interfaces":[{"interface":"back","average":{"1min":0.248,"5min":0.25700000000000001,"15min":0.251},"min":{"1min":0.17000000000000001,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"last":0.22700000000000001},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.27600000000000002,"15min":0.26800000000000002},"min":{"1min":0.192,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"last":0.20399999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:33:31.201 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:33:31.400+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)

Ignoring malparsed backtrace: 2023-03-25T04:33:47.531+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:33:47.951+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:33:48.505+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)

Ignoring malparsed backtrace: 2023-03-25T04:34:06.104 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json
, 2023-03-25T04:34:06.118+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:34:06.497 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:34:23.399 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:34:23.665+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:34:23.712 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:34:40.195+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:34:40.523+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:34:40.676 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace: 2023-03-25T04:35:03.381+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:35:04.139+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:35:04.391+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)

Ignoring malparsed backtrace: 2023-03-25T04:35:22.738 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean
, 2023-03-25T04:35:22.739 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}
, 2023-03-25T04:35:22.739 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json

Ignoring malparsed backtrace: 2023-03-25T04:35:41.393+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:35:42.352+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:35:42.376+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:35:57.384 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":340,"stamp":"2023-03-25T04:35:56.194992+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.001930"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":36,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:31:43.322894+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:31:43.322894+0000","last_peered":"2023-03-25T04:31:43.322894+0000","last_clean":"2023-03-25T04:31:43.322894+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:31:43.322894+0000","last_undegraded":"2023-03-25T04:31:43.322894+0000","last_fullsized":"2023-03-25T04:31:43.322894+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":28,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:31:43.322876+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:31:43.322876+0000","last_peered":"2023-03-25T04:31:43.322876+0000","last_clean":"2023-03-25T04:31:43.322876+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:31:43.322876+0000","last_undegraded":"2023-03-25T04:31:43.322876+0000","last_fullsized":"2023-03-25T04:31:43.322876+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:23.331998+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:33:23.331998+0000","last_peered":"2023-03-25T04:33:23.331998+0000","last_clean":"2023-03-25T04:33:23.331998+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:33:23.331998+0000","last_undegraded":"2023-03-25T04:33:23.331998+0000","last_fullsized":"2023-03-25T04:33:23.331998+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:30.576723+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:33:30.576723+0000","last_peered":"2023-03-25T04:33:30.576723+0000","last_clean":"2023-03-25T04:33:30.576723+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:33:30.576723+0000","last_undegraded":"2023-03-25T04:33:30.576723+0000","last_fullsized":"2023-03-25T04:33:30.576723+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":37,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:38.333447+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:33:38.333447+0000","last_peered":"2023-03-25T04:33:38.333447+0000","last_clean":"2023-03-25T04:33:38.333447+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:33:38.333447+0000","last_undegraded":"2023-03-25T04:33:38.333447+0000","last_fullsized":"2023-03-25T04:33:38.333447+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:38.333467+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:33:38.333467+0000","last_peered":"2023-03-25T04:33:38.333467+0000","last_clean":"2023-03-25T04:33:38.333467+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:33:38.333467+0000","last_undegraded":"2023-03-25T04:33:38.333467+0000","last_fullsized":"2023-03-25T04:33:38.333467+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117098,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:35:49 2023","interfaces":[{"interface":"back","average":{"1min":0.255,"5min":0.26000000000000001,"15min":0.28199999999999997},"min":{"1min":0.17899999999999999,"5min":0.16,"15min":0.16},"max":{"1min":0.36399999999999999,"5min":0.42599999999999999,"15min":0.45100000000000001},"last":0.26100000000000001},{"interface":"front","average":{"1min":0.26000000000000001,"5min":0.26700000000000002,"15min":0.28599999999999998},"min":{"1min":0.188,"5min":0.17999999999999999,"15min":0.17499999999999999},"max":{"1min":0.39100000000000001,"5min":0.45200000000000001,"15min":0.46300000000000002},"last":0.28399999999999997}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051689,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:35:42 2023","interfaces":[{"interface":"back","average":{"1min":0.29399999999999998,"5min":0.27200000000000002,"15min":0.25800000000000001},"min":{"1min":0.17299999999999999,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.41099999999999998,"5min":0.42699999999999999,"15min":0.42699999999999999},"last":0.20399999999999999},{"interface":"front","average":{"1min":0.30299999999999999,"5min":0.28399999999999997,"15min":0.27300000000000002},"min":{"1min":0.19700000000000001,"5min":0.192,"15min":0.16600000000000001},"max":{"1min":0.43099999999999999,"5min":0.45800000000000002,"15min":0.45800000000000002},"last":0.22}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:35:57.624+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)

Ignoring malparsed backtrace: 2023-03-25T04:36:17.226 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:36:17.239 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":350,"stamp":"2023-03-25T04:36:16.198866+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002148"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":37,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:53.335026+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:33:53.335026+0000","last_peered":"2023-03-25T04:33:53.335026+0000","last_clean":"2023-03-25T04:33:53.335026+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:33:53.335026+0000","last_undegraded":"2023-03-25T04:33:53.335026+0000","last_fullsized":"2023-03-25T04:33:53.335026+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:53.335009+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:33:53.335009+0000","last_peered":"2023-03-25T04:33:53.335009+0000","last_clean":"2023-03-25T04:33:53.335009+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:33:53.335009+0000","last_undegraded":"2023-03-25T04:33:53.335009+0000","last_fullsized":"2023-03-25T04:33:53.335009+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:23.331998+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:33:23.331998+0000","last_peered":"2023-03-25T04:33:23.331998+0000","last_clean":"2023-03-25T04:33:23.331998+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:33:23.331998+0000","last_undegraded":"2023-03-25T04:33:23.331998+0000","last_fullsized":"2023-03-25T04:33:23.331998+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:30.576723+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:33:30.576723+0000","last_peered":"2023-03-25T04:33:30.576723+0000","last_clean":"2023-03-25T04:33:30.576723+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:33:30.576723+0000","last_undegraded":"2023-03-25T04:33:30.576723+0000","last_fullsized":"2023-03-25T04:33:30.576723+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":37,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:38.333447+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:33:38.333447+0000","last_peered":"2023-03-25T04:33:38.333447+0000","last_clean":"2023-03-25T04:33:38.333447+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:33:38.333447+0000","last_undegraded":"2023-03-25T04:33:38.333447+0000","last_fullsized":"2023-03-25T04:33:38.333447+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:38.333467+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:33:38.333467+0000","last_peered":"2023-03-25T04:33:38.333467+0000","last_clean":"2023-03-25T04:33:38.333467+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:33:38.333467+0000","last_undegraded":"2023-03-25T04:33:38.333467+0000","last_fullsized":"2023-03-25T04:33:38.333467+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117102,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:35:49 2023","interfaces":[{"interface":"back","average":{"1min":0.255,"5min":0.26000000000000001,"15min":0.28199999999999997},"min":{"1min":0.17899999999999999,"5min":0.16,"15min":0.16},"max":{"1min":0.36399999999999999,"5min":0.42599999999999999,"15min":0.45100000000000001},"last":0.28399999999999997},{"interface":"front","average":{"1min":0.26000000000000001,"5min":0.26700000000000002,"15min":0.28599999999999998},"min":{"1min":0.188,"5min":0.17999999999999999,"15min":0.17499999999999999},"max":{"1min":0.39100000000000001,"5min":0.45200000000000001,"15min":0.46300000000000002},"last":0.30299999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051693,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:35:42 2023","interfaces":[{"interface":"back","average":{"1min":0.29399999999999998,"5min":0.27200000000000002,"15min":0.25800000000000001},"min":{"1min":0.17299999999999999,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.41099999999999998,"5min":0.42699999999999999,"15min":0.42699999999999999},"last":0.35099999999999998},{"interface":"front","average":{"1min":0.30299999999999999,"5min":0.28399999999999997,"15min":0.27300000000000002},"min":{"1min":0.19700000000000001,"5min":0.192,"15min":0.16600000000000001},"max":{"1min":0.43099999999999999,"5min":0.45800000000000002,"15min":0.45800000000000002},"last":0.32900000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:36:34.556+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:36:34.717+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:36:35.583+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)

Ignoring malparsed backtrace: 2023-03-25T04:36:52.680 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json
, 2023-03-25T04:36:52.701+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:36:53.067 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:37:09.963 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:37:10.273 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all

Ignoring malparsed backtrace: 2023-03-25T04:37:26.856+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:37:27.288 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:37:27.538+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:37:41.903 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":392,"stamp":"2023-03-25T04:37:40.213373+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002012"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":37,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:53.335026+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:33:53.335026+0000","last_peered":"2023-03-25T04:33:53.335026+0000","last_clean":"2023-03-25T04:33:53.335026+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:33:53.335026+0000","last_undegraded":"2023-03-25T04:33:53.335026+0000","last_fullsized":"2023-03-25T04:33:53.335026+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:53.335009+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:33:53.335009+0000","last_peered":"2023-03-25T04:33:53.335009+0000","last_clean":"2023-03-25T04:33:53.335009+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:33:53.335009+0000","last_undegraded":"2023-03-25T04:33:53.335009+0000","last_fullsized":"2023-03-25T04:33:53.335009+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:23.343585+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:35:23.343585+0000","last_peered":"2023-03-25T04:35:23.343585+0000","last_clean":"2023-03-25T04:35:23.343585+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:35:23.343585+0000","last_undegraded":"2023-03-25T04:35:23.343585+0000","last_fullsized":"2023-03-25T04:35:23.343585+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:30.576723+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:33:30.576723+0000","last_peered":"2023-03-25T04:33:30.576723+0000","last_clean":"2023-03-25T04:33:30.576723+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:33:30.576723+0000","last_undegraded":"2023-03-25T04:33:30.576723+0000","last_fullsized":"2023-03-25T04:33:30.576723+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":37,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:38.333447+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:33:38.333447+0000","last_peered":"2023-03-25T04:33:38.333447+0000","last_clean":"2023-03-25T04:33:38.333447+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:33:38.333447+0000","last_undegraded":"2023-03-25T04:33:38.333447+0000","last_fullsized":"2023-03-25T04:33:38.333447+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:38.333467+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:33:38.333467+0000","last_peered":"2023-03-25T04:33:38.333467+0000","last_clean":"2023-03-25T04:33:38.333467+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:33:38.333467+0000","last_undegraded":"2023-03-25T04:33:38.333467+0000","last_fullsized":"2023-03-25T04:33:38.333467+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117119,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:36:50 2023","interfaces":[{"interface":"back","average":{"1min":0.254,"5min":0.25600000000000001,"15min":0.27900000000000003},"min":{"1min":0.20499999999999999,"5min":0.16,"15min":0.16},"max":{"1min":0.371,"5min":0.42599999999999999,"15min":0.45100000000000001},"last":0.23000000000000001},{"interface":"front","average":{"1min":0.26800000000000002,"5min":0.26500000000000001,"15min":0.28399999999999997},"min":{"1min":0.19700000000000001,"5min":0.17999999999999999,"15min":0.17499999999999999},"max":{"1min":0.34699999999999998,"5min":0.45200000000000001,"15min":0.46300000000000002},"last":0.20899999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051710,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:36:44 2023","interfaces":[{"interface":"back","average":{"1min":0.28399999999999997,"5min":0.27700000000000002,"15min":0.26100000000000001},"min":{"1min":0.192,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.40899999999999997,"5min":0.42699999999999999,"15min":0.42699999999999999},"last":0.39700000000000002},{"interface":"front","average":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.27500000000000002},"min":{"1min":0.182,"5min":0.182,"15min":0.16600000000000001},"max":{"1min":0.439,"5min":0.45800000000000002,"15min":0.45800000000000002},"last":0.439}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:37:42.400+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)

Ignoring malparsed backtrace: 2023-03-25T04:38:04.766+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:38:04.922 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:38:05.246 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:38:19.605 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":411,"stamp":"2023-03-25T04:38:18.220966+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002368"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":38,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:36:03.347070+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:36:03.347070+0000","last_peered":"2023-03-25T04:36:03.347070+0000","last_clean":"2023-03-25T04:36:03.347070+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:36:03.347070+0000","last_undegraded":"2023-03-25T04:36:03.347070+0000","last_fullsized":"2023-03-25T04:36:03.347070+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:36:03.347025+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:36:03.347025+0000","last_peered":"2023-03-25T04:36:03.347025+0000","last_clean":"2023-03-25T04:36:03.347025+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:36:03.347025+0000","last_undegraded":"2023-03-25T04:36:03.347025+0000","last_fullsized":"2023-03-25T04:36:03.347025+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:23.343585+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:35:23.343585+0000","last_peered":"2023-03-25T04:35:23.343585+0000","last_clean":"2023-03-25T04:35:23.343585+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:35:23.343585+0000","last_undegraded":"2023-03-25T04:35:23.343585+0000","last_fullsized":"2023-03-25T04:35:23.343585+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:35.587025+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:35:35.587025+0000","last_peered":"2023-03-25T04:35:35.587025+0000","last_clean":"2023-03-25T04:35:35.587025+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:35:35.587025+0000","last_undegraded":"2023-03-25T04:35:35.587025+0000","last_fullsized":"2023-03-25T04:35:35.587025+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":38,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:43.345308+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:35:43.345308+0000","last_peered":"2023-03-25T04:35:43.345308+0000","last_clean":"2023-03-25T04:35:43.345308+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:35:43.345308+0000","last_undegraded":"2023-03-25T04:35:43.345308+0000","last_fullsized":"2023-03-25T04:35:43.345308+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:43.345324+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:35:43.345324+0000","last_peered":"2023-03-25T04:35:43.345324+0000","last_clean":"2023-03-25T04:35:43.345324+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:35:43.345324+0000","last_undegraded":"2023-03-25T04:35:43.345324+0000","last_fullsized":"2023-03-25T04:35:43.345324+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117127,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:37:53 2023","interfaces":[{"interface":"back","average":{"1min":0.25800000000000001,"5min":0.253,"15min":0.27700000000000002},"min":{"1min":0.184,"5min":0.16,"15min":0.16},"max":{"1min":0.44600000000000001,"5min":0.44600000000000001,"15min":0.45100000000000001},"last":0.35199999999999998},{"interface":"front","average":{"1min":0.26900000000000002,"5min":0.26500000000000001,"15min":0.28199999999999997},"min":{"1min":0.192,"5min":0.17999999999999999,"15min":0.17499999999999999},"max":{"1min":0.46400000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"last":0.33200000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051717,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:37:44 2023","interfaces":[{"interface":"back","average":{"1min":0.27300000000000002,"5min":0.28199999999999997,"15min":0.26200000000000001},"min":{"1min":0.17899999999999999,"5min":0.17299999999999999,"15min":0.17000000000000001},"max":{"1min":0.41799999999999998,"5min":0.42099999999999999,"15min":0.42699999999999999},"last":0.28699999999999998},{"interface":"front","average":{"1min":0.29399999999999998,"5min":0.29099999999999998,"15min":0.27700000000000002},"min":{"1min":0.19900000000000001,"5min":0.182,"15min":0.16600000000000001},"max":{"1min":0.441,"5min":0.441,"15min":0.45800000000000002},"last":0.26300000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:38:19.718+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:38:20.315+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)

Ignoring malparsed backtrace: 2023-03-25T04:38:44.283 INFO:teuthology.orchestra.run.smithi070.stdout:
, 2023-03-25T04:38:44.283 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":781,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":16,"modified":"2023-03-25T04:38:40.225321+0000","services":{}},"progress_events":{}}
, 2023-03-25T04:38:44.296 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now

Ignoring malparsed backtrace: 2023-03-25T04:39:02.477+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:39:02.578+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:39:03.524+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:39:20.359 INFO:teuthology.orchestra.run.smithi070.stdout:
, 2023-03-25T04:39:20.359 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":818,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":16,"modified":"2023-03-25T04:38:40.225321+0000","services":{}},"progress_events":{}}
, 2023-03-25T04:39:20.372 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now

Ignoring malparsed backtrace: 2023-03-25T04:39:37.527 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:39:37.541 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":450,"stamp":"2023-03-25T04:39:36.235332+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002086"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":38,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:36:03.347070+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:36:03.347070+0000","last_peered":"2023-03-25T04:36:03.347070+0000","last_clean":"2023-03-25T04:36:03.347070+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:36:03.347070+0000","last_undegraded":"2023-03-25T04:36:03.347070+0000","last_fullsized":"2023-03-25T04:36:03.347070+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:36:03.347025+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:36:03.347025+0000","last_peered":"2023-03-25T04:36:03.347025+0000","last_clean":"2023-03-25T04:36:03.347025+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:36:03.347025+0000","last_undegraded":"2023-03-25T04:36:03.347025+0000","last_fullsized":"2023-03-25T04:36:03.347025+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:23.353778+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:37:23.353778+0000","last_peered":"2023-03-25T04:37:23.353778+0000","last_clean":"2023-03-25T04:37:23.353778+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:37:23.353778+0000","last_undegraded":"2023-03-25T04:37:23.353778+0000","last_fullsized":"2023-03-25T04:37:23.353778+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:35.587025+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:35:35.587025+0000","last_peered":"2023-03-25T04:35:35.587025+0000","last_clean":"2023-03-25T04:35:35.587025+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:35:35.587025+0000","last_undegraded":"2023-03-25T04:35:35.587025+0000","last_fullsized":"2023-03-25T04:35:35.587025+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":38,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:43.345308+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:35:43.345308+0000","last_peered":"2023-03-25T04:35:43.345308+0000","last_clean":"2023-03-25T04:35:43.345308+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:35:43.345308+0000","last_undegraded":"2023-03-25T04:35:43.345308+0000","last_fullsized":"2023-03-25T04:35:43.345308+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:43.345324+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:35:43.345324+0000","last_peered":"2023-03-25T04:35:43.345324+0000","last_clean":"2023-03-25T04:35:43.345324+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:35:43.345324+0000","last_undegraded":"2023-03-25T04:35:43.345324+0000","last_fullsized":"2023-03-25T04:35:43.345324+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117143,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:38:54 2023","interfaces":[{"interface":"back","average":{"1min":0.25800000000000001,"5min":0.25700000000000001,"15min":0.27400000000000002},"min":{"1min":0.192,"5min":0.17399999999999999,"15min":0.16},"max":{"1min":0.39400000000000002,"5min":0.44600000000000001,"15min":0.45100000000000001},"last":0.26500000000000001},{"interface":"front","average":{"1min":0.25900000000000001,"5min":0.26700000000000002,"15min":0.27900000000000003},"min":{"1min":0.19500000000000001,"5min":0.188,"15min":0.17499999999999999},"max":{"1min":0.41499999999999998,"5min":0.46400000000000002,"15min":0.46400000000000002},"last":0.29099999999999998}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051733,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:38:45 2023","interfaces":[{"interface":"back","average":{"1min":0.27300000000000002,"5min":0.28100000000000003,"15min":0.26400000000000001},"min":{"1min":0.19400000000000001,"5min":0.17299999999999999,"15min":0.17000000000000001},"max":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"last":0.27300000000000002},{"interface":"front","average":{"1min":0.28799999999999998,"5min":0.29099999999999998,"15min":0.27800000000000002},"min":{"1min":0.19800000000000001,"5min":0.182,"15min":0.16600000000000001},"max":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"last":0.308}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:39:54.416+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:39:54.455 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:39:54.780 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:40:09.288+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:40:09.389+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:40:10.267+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)

Ignoring malparsed backtrace: 2023-03-25T04:40:28.984 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":476,"stamp":"2023-03-25T04:40:28.245911+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002693"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":39,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:38:13.358968+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:38:13.358968+0000","last_peered":"2023-03-25T04:38:13.358968+0000","last_clean":"2023-03-25T04:38:13.358968+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:38:13.358968+0000","last_undegraded":"2023-03-25T04:38:13.358968+0000","last_fullsized":"2023-03-25T04:38:13.358968+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:38:13.358922+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:38:13.358922+0000","last_peered":"2023-03-25T04:38:13.358922+0000","last_clean":"2023-03-25T04:38:13.358922+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:38:13.358922+0000","last_undegraded":"2023-03-25T04:38:13.358922+0000","last_fullsized":"2023-03-25T04:38:13.358922+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:23.353778+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:37:23.353778+0000","last_peered":"2023-03-25T04:37:23.353778+0000","last_clean":"2023-03-25T04:37:23.353778+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:37:23.353778+0000","last_undegraded":"2023-03-25T04:37:23.353778+0000","last_fullsized":"2023-03-25T04:37:23.353778+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:40.598579+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:37:40.598579+0000","last_peered":"2023-03-25T04:37:40.598579+0000","last_clean":"2023-03-25T04:37:40.598579+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:37:40.598579+0000","last_undegraded":"2023-03-25T04:37:40.598579+0000","last_fullsized":"2023-03-25T04:37:40.598579+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":39,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:48.356344+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:37:48.356344+0000","last_peered":"2023-03-25T04:37:48.356344+0000","last_clean":"2023-03-25T04:37:48.356344+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:37:48.356344+0000","last_undegraded":"2023-03-25T04:37:48.356344+0000","last_fullsized":"2023-03-25T04:37:48.356344+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:48.356365+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:37:48.356365+0000","last_peered":"2023-03-25T04:37:48.356365+0000","last_clean":"2023-03-25T04:37:48.356365+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:37:48.356365+0000","last_undegraded":"2023-03-25T04:37:48.356365+0000","last_fullsized":"2023-03-25T04:37:48.356365+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117153,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:39:54 2023","interfaces":[{"interface":"back","average":{"1min":0.25900000000000001,"5min":0.25600000000000001,"15min":0.27100000000000002},"min":{"1min":0.188,"5min":0.17899999999999999,"15min":0.16},"max":{"1min":0.41999999999999998,"5min":0.44600000000000001,"15min":0.45100000000000001},"last":0.255},{"interface":"front","average":{"1min":0.27500000000000002,"5min":0.26600000000000001,"15min":0.27700000000000002},"min":{"1min":0.17699999999999999,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.45600000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"last":0.26800000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051743,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:39:47 2023","interfaces":[{"interface":"back","average":{"1min":0.26300000000000001,"5min":0.27700000000000002,"15min":0.26500000000000001},"min":{"1min":0.187,"5min":0.17299999999999999,"15min":0.17000000000000001},"max":{"1min":0.433,"5min":0.433,"15min":0.433},"last":0.36599999999999999},{"interface":"front","average":{"1min":0.27100000000000002,"5min":0.28799999999999998,"15min":0.27900000000000003},"min":{"1min":0.20100000000000001,"5min":0.182,"15min":0.16600000000000001},"max":{"1min":0.48599999999999999,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.34999999999999998}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:40:28.986 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean
, 2023-03-25T04:40:28.986 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}

Ignoring malparsed backtrace: 2023-03-25T04:40:47.229+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:40:47.503+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:40:48.251+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)

Ignoring malparsed backtrace: 2023-03-25T04:41:03.267 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:41:03.578 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all

Ignoring malparsed backtrace: 2023-03-25T04:41:21.347 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":502,"stamp":"2023-03-25T04:41:20.256204+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002342"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":39,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:38:13.358968+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:38:13.358968+0000","last_peered":"2023-03-25T04:38:13.358968+0000","last_clean":"2023-03-25T04:38:13.358968+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:38:13.358968+0000","last_undegraded":"2023-03-25T04:38:13.358968+0000","last_fullsized":"2023-03-25T04:38:13.358968+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:38:13.358922+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:38:13.358922+0000","last_peered":"2023-03-25T04:38:13.358922+0000","last_clean":"2023-03-25T04:38:13.358922+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:38:13.358922+0000","last_undegraded":"2023-03-25T04:38:13.358922+0000","last_fullsized":"2023-03-25T04:38:13.358922+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:23.353778+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:37:23.353778+0000","last_peered":"2023-03-25T04:37:23.353778+0000","last_clean":"2023-03-25T04:37:23.353778+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:37:23.353778+0000","last_undegraded":"2023-03-25T04:37:23.353778+0000","last_fullsized":"2023-03-25T04:37:23.353778+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:40.598579+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:37:40.598579+0000","last_peered":"2023-03-25T04:37:40.598579+0000","last_clean":"2023-03-25T04:37:40.598579+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:37:40.598579+0000","last_undegraded":"2023-03-25T04:37:40.598579+0000","last_fullsized":"2023-03-25T04:37:40.598579+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":39,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:48.356344+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:37:48.356344+0000","last_peered":"2023-03-25T04:37:48.356344+0000","last_clean":"2023-03-25T04:37:48.356344+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:37:48.356344+0000","last_undegraded":"2023-03-25T04:37:48.356344+0000","last_fullsized":"2023-03-25T04:37:48.356344+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:48.356365+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:37:48.356365+0000","last_peered":"2023-03-25T04:37:48.356365+0000","last_clean":"2023-03-25T04:37:48.356365+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:37:48.356365+0000","last_undegraded":"2023-03-25T04:37:48.356365+0000","last_fullsized":"2023-03-25T04:37:48.356365+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117163,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:40:56 2023","interfaces":[{"interface":"back","average":{"1min":0.28799999999999998,"5min":0.26300000000000001,"15min":0.27100000000000002},"min":{"1min":0.188,"5min":0.184,"15min":0.16},"max":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.17499999999999999},{"interface":"front","average":{"1min":0.29599999999999999,"5min":0.27300000000000002,"15min":0.27700000000000002},"min":{"1min":0.20599999999999999,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.43099999999999999,"5min":0.46400000000000002,"15min":0.46400000000000002},"last":0.20699999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051754,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:40:52 2023","interfaces":[{"interface":"back","average":{"1min":0.28599999999999998,"5min":0.27500000000000002,"15min":0.26700000000000002},"min":{"1min":0.17899999999999999,"5min":0.17899999999999999,"15min":0.17000000000000001},"max":{"1min":0.44,"5min":0.44,"15min":0.44},"last":0.23699999999999999},{"interface":"front","average":{"1min":0.29799999999999999,"5min":0.28699999999999998,"15min":0.28100000000000003},"min":{"1min":0.20300000000000001,"5min":0.182,"15min":0.16600000000000001},"max":{"1min":0.47599999999999998,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.28000000000000003}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:41:21.349 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean
, 2023-03-25T04:41:21.349 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}

Ignoring malparsed backtrace: 2023-03-25T04:41:39.867+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:41:40.383+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:41:40.905+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:41:58.219 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json
, 2023-03-25T04:41:58.319+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:41:58.644 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:42:18.193 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":530,"stamp":"2023-03-25T04:42:16.266659+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002356"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":39,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:38:13.358968+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:38:13.358968+0000","last_peered":"2023-03-25T04:38:13.358968+0000","last_clean":"2023-03-25T04:38:13.358968+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:38:13.358968+0000","last_undegraded":"2023-03-25T04:38:13.358968+0000","last_fullsized":"2023-03-25T04:38:13.358968+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:38:13.358922+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:38:13.358922+0000","last_peered":"2023-03-25T04:38:13.358922+0000","last_clean":"2023-03-25T04:38:13.358922+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:38:13.358922+0000","last_undegraded":"2023-03-25T04:38:13.358922+0000","last_fullsized":"2023-03-25T04:38:13.358922+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:23.364593+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:39:23.364593+0000","last_peered":"2023-03-25T04:39:23.364593+0000","last_clean":"2023-03-25T04:39:23.364593+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:39:23.364593+0000","last_undegraded":"2023-03-25T04:39:23.364593+0000","last_fullsized":"2023-03-25T04:39:23.364593+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:45.608475+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:39:45.608475+0000","last_peered":"2023-03-25T04:39:45.608475+0000","last_clean":"2023-03-25T04:39:45.608475+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:39:45.608475+0000","last_undegraded":"2023-03-25T04:39:45.608475+0000","last_fullsized":"2023-03-25T04:39:45.608475+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":40,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:53.367398+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:39:53.367398+0000","last_peered":"2023-03-25T04:39:53.367398+0000","last_clean":"2023-03-25T04:39:53.367398+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:39:53.367398+0000","last_undegraded":"2023-03-25T04:39:53.367398+0000","last_fullsized":"2023-03-25T04:39:53.367398+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:53.367424+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:39:53.367424+0000","last_peered":"2023-03-25T04:39:53.367424+0000","last_clean":"2023-03-25T04:39:53.367424+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:39:53.367424+0000","last_undegraded":"2023-03-25T04:39:53.367424+0000","last_fullsized":"2023-03-25T04:39:53.367424+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117175,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:41:58 2023","interfaces":[{"interface":"back","average":{"1min":0.25800000000000001,"5min":0.26400000000000001,"15min":0.26800000000000002},"min":{"1min":0.17499999999999999,"5min":0.17499999999999999,"15min":0.16},"max":{"1min":0.39100000000000001,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.22900000000000001},{"interface":"front","average":{"1min":0.26300000000000001,"5min":0.27200000000000002,"15min":0.27400000000000002},"min":{"1min":0.20599999999999999,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.35699999999999998,"5min":0.46400000000000002,"15min":0.46400000000000002},"last":0.26400000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051765,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:41:52 2023","interfaces":[{"interface":"back","average":{"1min":0.26500000000000001,"5min":0.27200000000000002,"15min":0.26800000000000002},"min":{"1min":0.17000000000000001,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.39200000000000002,"5min":0.44,"15min":0.44},"last":0.23899999999999999},{"interface":"front","average":{"1min":0.28100000000000003,"5min":0.28599999999999998,"15min":0.28199999999999997},"min":{"1min":0.19400000000000001,"5min":0.19400000000000001,"15min":0.16600000000000001},"max":{"1min":0.41399999999999998,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.254}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:42:18.297+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:42:18.958+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:42:35.512 INFO:teuthology.orchestra.run.smithi070.stdout:
, 2023-03-25T04:42:35.512 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":1013,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":17,"modified":"2023-03-25T04:41:54.262744+0000","services":{}},"progress_events":{}}
, 2023-03-25T04:42:35.525 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:42:52.741 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":548,"stamp":"2023-03-25T04:42:52.273601+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002221"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":40,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:40:23.370056+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:40:23.370056+0000","last_peered":"2023-03-25T04:40:23.370056+0000","last_clean":"2023-03-25T04:40:23.370056+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:40:23.370056+0000","last_undegraded":"2023-03-25T04:40:23.370056+0000","last_fullsized":"2023-03-25T04:40:23.370056+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:40:23.370039+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:40:23.370039+0000","last_peered":"2023-03-25T04:40:23.370039+0000","last_clean":"2023-03-25T04:40:23.370039+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:40:23.370039+0000","last_undegraded":"2023-03-25T04:40:23.370039+0000","last_fullsized":"2023-03-25T04:40:23.370039+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:23.364593+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:39:23.364593+0000","last_peered":"2023-03-25T04:39:23.364593+0000","last_clean":"2023-03-25T04:39:23.364593+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:39:23.364593+0000","last_undegraded":"2023-03-25T04:39:23.364593+0000","last_fullsized":"2023-03-25T04:39:23.364593+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:45.608475+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:39:45.608475+0000","last_peered":"2023-03-25T04:39:45.608475+0000","last_clean":"2023-03-25T04:39:45.608475+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:39:45.608475+0000","last_undegraded":"2023-03-25T04:39:45.608475+0000","last_fullsized":"2023-03-25T04:39:45.608475+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":40,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:53.367398+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:39:53.367398+0000","last_peered":"2023-03-25T04:39:53.367398+0000","last_clean":"2023-03-25T04:39:53.367398+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:39:53.367398+0000","last_undegraded":"2023-03-25T04:39:53.367398+0000","last_fullsized":"2023-03-25T04:39:53.367398+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:53.367424+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:39:53.367424+0000","last_peered":"2023-03-25T04:39:53.367424+0000","last_clean":"2023-03-25T04:39:53.367424+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:39:53.367424+0000","last_undegraded":"2023-03-25T04:39:53.367424+0000","last_fullsized":"2023-03-25T04:39:53.367424+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117182,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:41:58 2023","interfaces":[{"interface":"back","average":{"1min":0.25800000000000001,"5min":0.26400000000000001,"15min":0.26800000000000002},"min":{"1min":0.17499999999999999,"5min":0.17499999999999999,"15min":0.16},"max":{"1min":0.39100000000000001,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.19600000000000001},{"interface":"front","average":{"1min":0.26300000000000001,"5min":0.27200000000000002,"15min":0.27400000000000002},"min":{"1min":0.20599999999999999,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.35699999999999998,"5min":0.46400000000000002,"15min":0.46400000000000002},"last":0.219}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051773,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:41:52 2023","interfaces":[{"interface":"back","average":{"1min":0.26500000000000001,"5min":0.27200000000000002,"15min":0.26800000000000002},"min":{"1min":0.17000000000000001,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.39200000000000002,"5min":0.44,"15min":0.44},"last":0.19},{"interface":"front","average":{"1min":0.28100000000000003,"5min":0.28599999999999998,"15min":0.28199999999999997},"min":{"1min":0.19400000000000001,"5min":0.19400000000000001,"15min":0.16600000000000001},"max":{"1min":0.41399999999999998,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.17599999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:42:52.742 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean

Ignoring malparsed backtrace: 2023-03-25T04:43:09.973 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:43:09.986 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":556,"stamp":"2023-03-25T04:43:08.276556+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002359"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":40,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:40:23.370056+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:40:23.370056+0000","last_peered":"2023-03-25T04:40:23.370056+0000","last_clean":"2023-03-25T04:40:23.370056+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:40:23.370056+0000","last_undegraded":"2023-03-25T04:40:23.370056+0000","last_fullsized":"2023-03-25T04:40:23.370056+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:40:23.370039+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:40:23.370039+0000","last_peered":"2023-03-25T04:40:23.370039+0000","last_clean":"2023-03-25T04:40:23.370039+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:40:23.370039+0000","last_undegraded":"2023-03-25T04:40:23.370039+0000","last_fullsized":"2023-03-25T04:40:23.370039+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:23.364593+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:39:23.364593+0000","last_peered":"2023-03-25T04:39:23.364593+0000","last_clean":"2023-03-25T04:39:23.364593+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:39:23.364593+0000","last_undegraded":"2023-03-25T04:39:23.364593+0000","last_fullsized":"2023-03-25T04:39:23.364593+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:45.608475+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:39:45.608475+0000","last_peered":"2023-03-25T04:39:45.608475+0000","last_clean":"2023-03-25T04:39:45.608475+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:39:45.608475+0000","last_undegraded":"2023-03-25T04:39:45.608475+0000","last_fullsized":"2023-03-25T04:39:45.608475+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":40,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:53.367398+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:39:53.367398+0000","last_peered":"2023-03-25T04:39:53.367398+0000","last_clean":"2023-03-25T04:39:53.367398+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:39:53.367398+0000","last_undegraded":"2023-03-25T04:39:53.367398+0000","last_fullsized":"2023-03-25T04:39:53.367398+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:53.367424+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:39:53.367424+0000","last_peered":"2023-03-25T04:39:53.367424+0000","last_clean":"2023-03-25T04:39:53.367424+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:39:53.367424+0000","last_undegraded":"2023-03-25T04:39:53.367424+0000","last_fullsized":"2023-03-25T04:39:53.367424+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117185,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:43:00 2023","interfaces":[{"interface":"back","average":{"1min":0.27400000000000002,"5min":0.26700000000000002,"15min":0.26600000000000001},"min":{"1min":0.19600000000000001,"5min":0.17499999999999999,"15min":0.16},"max":{"1min":0.42999999999999999,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.27800000000000002},{"interface":"front","average":{"1min":0.28199999999999997,"5min":0.27500000000000002,"15min":0.27300000000000002},"min":{"1min":0.185,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.47199999999999998,"5min":0.47199999999999998,"15min":0.47199999999999998},"last":0.247}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051776,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:42:54 2023","interfaces":[{"interface":"back","average":{"1min":0.26300000000000001,"5min":0.27000000000000002,"15min":0.26900000000000002},"min":{"1min":0.17399999999999999,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.438,"5min":0.44,"15min":0.44},"last":0.20100000000000001},{"interface":"front","average":{"1min":0.27400000000000002,"5min":0.28199999999999997,"15min":0.28299999999999997},"min":{"1min":0.17599999999999999,"5min":0.17599999999999999,"15min":0.16600000000000001},"max":{"1min":0.41599999999999998,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.23499999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:43:25.042+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:43:25.411+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:43:26.013+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:43:47.937+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:43:48.534+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:43:48.889+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:44:09.318 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:44:09.331 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":586,"stamp":"2023-03-25T04:44:08.288933+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002697"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":40,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:40:23.370056+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:40:23.370056+0000","last_peered":"2023-03-25T04:40:23.370056+0000","last_clean":"2023-03-25T04:40:23.370056+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:40:23.370056+0000","last_undegraded":"2023-03-25T04:40:23.370056+0000","last_fullsized":"2023-03-25T04:40:23.370056+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:40:23.370039+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:40:23.370039+0000","last_peered":"2023-03-25T04:40:23.370039+0000","last_clean":"2023-03-25T04:40:23.370039+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:40:23.370039+0000","last_undegraded":"2023-03-25T04:40:23.370039+0000","last_fullsized":"2023-03-25T04:40:23.370039+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:23.375106+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:41:23.375106+0000","last_peered":"2023-03-25T04:41:23.375106+0000","last_clean":"2023-03-25T04:41:23.375106+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:41:23.375106+0000","last_undegraded":"2023-03-25T04:41:23.375106+0000","last_fullsized":"2023-03-25T04:41:23.375106+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:50.618930+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:41:50.618930+0000","last_peered":"2023-03-25T04:41:50.618930+0000","last_clean":"2023-03-25T04:41:50.618930+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:41:50.618930+0000","last_undegraded":"2023-03-25T04:41:50.618930+0000","last_fullsized":"2023-03-25T04:41:50.618930+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":41,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:58.377821+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:41:58.377821+0000","last_peered":"2023-03-25T04:41:58.377821+0000","last_clean":"2023-03-25T04:41:58.377821+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:41:58.377821+0000","last_undegraded":"2023-03-25T04:41:58.377821+0000","last_fullsized":"2023-03-25T04:41:58.377821+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:58.377836+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:41:58.377836+0000","last_peered":"2023-03-25T04:41:58.377836+0000","last_clean":"2023-03-25T04:41:58.377836+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:41:58.377836+0000","last_undegraded":"2023-03-25T04:41:58.377836+0000","last_fullsized":"2023-03-25T04:41:58.377836+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117197,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:44:01 2023","interfaces":[{"interface":"back","average":{"1min":0.27200000000000002,"5min":0.27000000000000002,"15min":0.26400000000000001},"min":{"1min":0.20399999999999999,"5min":0.17499999999999999,"15min":0.16},"max":{"1min":0.38400000000000001,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.245},{"interface":"front","average":{"1min":0.27600000000000002,"5min":0.27800000000000002,"15min":0.27100000000000002},"min":{"1min":0.23100000000000001,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.40400000000000003,"5min":0.47199999999999998,"15min":0.47199999999999998},"last":0.27500000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051788,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:43:55 2023","interfaces":[{"interface":"back","average":{"1min":0.26100000000000001,"5min":0.26700000000000002,"15min":0.27000000000000002},"min":{"1min":0.17799999999999999,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.374,"5min":0.44,"15min":0.44},"last":0.30299999999999999},{"interface":"front","average":{"1min":0.28299999999999997,"5min":0.28100000000000003,"15min":0.28399999999999997},"min":{"1min":0.20899999999999999,"5min":0.17599999999999999,"15min":0.16600000000000001},"max":{"1min":0.40100000000000002,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.27800000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:44:28.923 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":1126,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":19,"modified":"2023-03-25T04:43:20.278959+0000","services":{}},"progress_events":{}}
, 2023-03-25T04:44:28.936 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now
, 2023-03-25T04:44:28.937 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace: 2023-03-25T04:44:48.786 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:44:48.800 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":606,"stamp":"2023-03-25T04:44:48.296805+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002182"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":41,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:42:34.382196+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:42:34.382196+0000","last_peered":"2023-03-25T04:42:34.382196+0000","last_clean":"2023-03-25T04:42:34.382196+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:42:34.382196+0000","last_undegraded":"2023-03-25T04:42:34.382196+0000","last_fullsized":"2023-03-25T04:42:34.382196+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:42:29.381744+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:42:29.381744+0000","last_peered":"2023-03-25T04:42:29.381744+0000","last_clean":"2023-03-25T04:42:29.381744+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:42:29.381744+0000","last_undegraded":"2023-03-25T04:42:29.381744+0000","last_fullsized":"2023-03-25T04:42:29.381744+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:23.375106+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:41:23.375106+0000","last_peered":"2023-03-25T04:41:23.375106+0000","last_clean":"2023-03-25T04:41:23.375106+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:41:23.375106+0000","last_undegraded":"2023-03-25T04:41:23.375106+0000","last_fullsized":"2023-03-25T04:41:23.375106+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:50.618930+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:41:50.618930+0000","last_peered":"2023-03-25T04:41:50.618930+0000","last_clean":"2023-03-25T04:41:50.618930+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:41:50.618930+0000","last_undegraded":"2023-03-25T04:41:50.618930+0000","last_fullsized":"2023-03-25T04:41:50.618930+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":41,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:58.377821+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:41:58.377821+0000","last_peered":"2023-03-25T04:41:58.377821+0000","last_clean":"2023-03-25T04:41:58.377821+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:41:58.377821+0000","last_undegraded":"2023-03-25T04:41:58.377821+0000","last_fullsized":"2023-03-25T04:41:58.377821+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:58.377836+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:41:58.377836+0000","last_peered":"2023-03-25T04:41:58.377836+0000","last_clean":"2023-03-25T04:41:58.377836+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:41:58.377836+0000","last_undegraded":"2023-03-25T04:41:58.377836+0000","last_fullsized":"2023-03-25T04:41:58.377836+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117206,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:44:01 2023","interfaces":[{"interface":"back","average":{"1min":0.27200000000000002,"5min":0.27000000000000002,"15min":0.26400000000000001},"min":{"1min":0.20399999999999999,"5min":0.17499999999999999,"15min":0.16},"max":{"1min":0.38400000000000001,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.216},{"interface":"front","average":{"1min":0.27600000000000002,"5min":0.27800000000000002,"15min":0.27100000000000002},"min":{"1min":0.23100000000000001,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.40400000000000003,"5min":0.47199999999999998,"15min":0.47199999999999998},"last":0.23999999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051796,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:43:55 2023","interfaces":[{"interface":"back","average":{"1min":0.26100000000000001,"5min":0.26700000000000002,"15min":0.27000000000000002},"min":{"1min":0.17799999999999999,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.374,"5min":0.44,"15min":0.44},"last":0.25600000000000001},{"interface":"front","average":{"1min":0.28299999999999997,"5min":0.28100000000000003,"15min":0.28399999999999997},"min":{"1min":0.20899999999999999,"5min":0.17599999999999999,"15min":0.16600000000000001},"max":{"1min":0.40100000000000002,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.28000000000000003}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:45:05.762 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:45:06.073 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all

Ignoring malparsed backtrace: 2023-03-25T04:45:25.658 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":624,"stamp":"2023-03-25T04:45:24.304294+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002366"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":41,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:42:34.382196+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:42:34.382196+0000","last_peered":"2023-03-25T04:42:34.382196+0000","last_clean":"2023-03-25T04:42:34.382196+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:42:34.382196+0000","last_undegraded":"2023-03-25T04:42:34.382196+0000","last_fullsized":"2023-03-25T04:42:34.382196+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:42:29.381744+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:42:29.381744+0000","last_peered":"2023-03-25T04:42:29.381744+0000","last_clean":"2023-03-25T04:42:29.381744+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:42:29.381744+0000","last_undegraded":"2023-03-25T04:42:29.381744+0000","last_fullsized":"2023-03-25T04:42:29.381744+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:23.375106+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:41:23.375106+0000","last_peered":"2023-03-25T04:41:23.375106+0000","last_clean":"2023-03-25T04:41:23.375106+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:41:23.375106+0000","last_undegraded":"2023-03-25T04:41:23.375106+0000","last_fullsized":"2023-03-25T04:41:23.375106+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:50.618930+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:41:50.618930+0000","last_peered":"2023-03-25T04:41:50.618930+0000","last_clean":"2023-03-25T04:41:50.618930+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:41:50.618930+0000","last_undegraded":"2023-03-25T04:41:50.618930+0000","last_fullsized":"2023-03-25T04:41:50.618930+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":41,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:58.377821+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:41:58.377821+0000","last_peered":"2023-03-25T04:41:58.377821+0000","last_clean":"2023-03-25T04:41:58.377821+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:41:58.377821+0000","last_undegraded":"2023-03-25T04:41:58.377821+0000","last_fullsized":"2023-03-25T04:41:58.377821+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:58.377836+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:41:58.377836+0000","last_peered":"2023-03-25T04:41:58.377836+0000","last_clean":"2023-03-25T04:41:58.377836+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:41:58.377836+0000","last_undegraded":"2023-03-25T04:41:58.377836+0000","last_fullsized":"2023-03-25T04:41:58.377836+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117213,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:45:01 2023","interfaces":[{"interface":"back","average":{"1min":0.251,"5min":0.26800000000000002,"15min":0.26400000000000001},"min":{"1min":0.183,"5min":0.17499999999999999,"15min":0.16},"max":{"1min":0.41499999999999998,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.26900000000000002},{"interface":"front","average":{"1min":0.28000000000000003,"5min":0.27900000000000003,"15min":0.27100000000000002},"min":{"1min":0.20999999999999999,"5min":0.185,"15min":0.17499999999999999},"max":{"1min":0.44400000000000001,"5min":0.47199999999999998,"15min":0.47199999999999998},"last":0.27900000000000003}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051803,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:44:55 2023","interfaces":[{"interface":"back","average":{"1min":0.252,"5min":0.26500000000000001,"15min":0.26900000000000002},"min":{"1min":0.183,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.39700000000000002,"5min":0.44,"15min":0.44},"last":0.20699999999999999},{"interface":"front","average":{"1min":0.27000000000000002,"5min":0.28100000000000003,"15min":0.28299999999999997},"min":{"1min":0.20300000000000001,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":0.42099999999999999,"5min":0.47599999999999998,"15min":0.48599999999999999},"last":0.23599999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:45:25.659 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:45:25.969 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:45:42.735+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:45:42.744+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:45:42.897 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace: 2023-03-25T04:46:00.183 INFO:teuthology.orchestra.run.smithi070.stdout:
, 2023-03-25T04:46:00.183 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":1217,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":20,"modified":"2023-03-25T04:45:42.307700+0000","services":{"mgr":{"daemons":{"summary":"","x":{"start_epoch":0,"start_stamp":"0.000000","gid":0,"addr":"(unrecognized address family 0)/0","metadata":{},"task_status":{}}}}}},"progress_events":{}}
, 2023-03-25T04:46:00.197 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now

Ignoring malparsed backtrace: 2023-03-25T04:46:19.910+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:46:20.039 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all

Ignoring malparsed backtrace: 2023-03-25T04:46:39.611 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}
, 2023-03-25T04:46:39.611 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json
, 2023-03-25T04:46:39.613+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:46:58.485+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:46:58.764+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:46:59.519+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:47:14.132 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:47:14.146 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":678,"stamp":"2023-03-25T04:47:12.325044+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002571"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":42,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:44.393426+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:44:44.393426+0000","last_peered":"2023-03-25T04:44:44.393426+0000","last_clean":"2023-03-25T04:44:44.393426+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:44:44.393426+0000","last_undegraded":"2023-03-25T04:44:44.393426+0000","last_fullsized":"2023-03-25T04:44:44.393426+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:39.393044+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:44:39.393044+0000","last_peered":"2023-03-25T04:44:39.393044+0000","last_clean":"2023-03-25T04:44:39.393044+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:44:39.393044+0000","last_undegraded":"2023-03-25T04:44:39.393044+0000","last_fullsized":"2023-03-25T04:44:39.393044+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:43:24.386177+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:43:24.386177+0000","last_peered":"2023-03-25T04:43:24.386177+0000","last_clean":"2023-03-25T04:43:24.386177+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:43:24.386177+0000","last_undegraded":"2023-03-25T04:43:24.386177+0000","last_fullsized":"2023-03-25T04:43:24.386177+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:43:55.629798+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:43:55.629798+0000","last_peered":"2023-03-25T04:43:55.629798+0000","last_clean":"2023-03-25T04:43:55.629798+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:43:55.629798+0000","last_undegraded":"2023-03-25T04:43:55.629798+0000","last_fullsized":"2023-03-25T04:43:55.629798+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":42,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:04.389904+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:44:04.389904+0000","last_peered":"2023-03-25T04:44:04.389904+0000","last_clean":"2023-03-25T04:44:04.389904+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:44:04.389904+0000","last_undegraded":"2023-03-25T04:44:04.389904+0000","last_fullsized":"2023-03-25T04:44:04.389904+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:04.389923+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:44:04.389923+0000","last_peered":"2023-03-25T04:44:04.389923+0000","last_clean":"2023-03-25T04:44:04.389923+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:44:04.389923+0000","last_undegraded":"2023-03-25T04:44:04.389923+0000","last_fullsized":"2023-03-25T04:44:04.389923+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117235,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:47:05 2023","interfaces":[{"interface":"back","average":{"1min":0.27800000000000002,"5min":0.27000000000000002,"15min":0.26300000000000001},"min":{"1min":0.20100000000000001,"5min":0.16900000000000001,"15min":0.16},"max":{"1min":0.39200000000000002,"5min":0.42999999999999999,"15min":0.45600000000000002},"last":0.379},{"interface":"front","average":{"1min":0.28399999999999997,"5min":0.28100000000000003,"15min":0.27200000000000002},"min":{"1min":0.17799999999999999,"5min":0.17799999999999999,"15min":0.17699999999999999},"max":{"1min":0.42599999999999999,"5min":0.47199999999999998,"15min":0.47199999999999998},"last":0.39800000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051825,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:47:01 2023","interfaces":[{"interface":"back","average":{"1min":0.27300000000000002,"5min":0.26100000000000001,"15min":0.27000000000000002},"min":{"1min":0.186,"5min":0.17399999999999999,"15min":0.17000000000000001},"max":{"1min":0.40400000000000003,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.23899999999999999},{"interface":"front","average":{"1min":0.28699999999999998,"5min":0.27500000000000002,"15min":0.28299999999999997},"min":{"1min":0.186,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.48599999999999999},"last":0.19600000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:47:31.198 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json
, 2023-03-25T04:47:31.411+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:47:31.789 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:47:49.509 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":696,"stamp":"2023-03-25T04:47:48.332093+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002697"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":42,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:44.393426+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:44:44.393426+0000","last_peered":"2023-03-25T04:44:44.393426+0000","last_clean":"2023-03-25T04:44:44.393426+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:44:44.393426+0000","last_undegraded":"2023-03-25T04:44:44.393426+0000","last_fullsized":"2023-03-25T04:44:44.393426+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:39.393044+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:44:39.393044+0000","last_peered":"2023-03-25T04:44:39.393044+0000","last_clean":"2023-03-25T04:44:39.393044+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:44:39.393044+0000","last_undegraded":"2023-03-25T04:44:39.393044+0000","last_fullsized":"2023-03-25T04:44:39.393044+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:45:24.396717+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:45:24.396717+0000","last_peered":"2023-03-25T04:45:24.396717+0000","last_clean":"2023-03-25T04:45:24.396717+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:45:24.396717+0000","last_undegraded":"2023-03-25T04:45:24.396717+0000","last_fullsized":"2023-03-25T04:45:24.396717+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:43:55.629798+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:43:55.629798+0000","last_peered":"2023-03-25T04:43:55.629798+0000","last_clean":"2023-03-25T04:43:55.629798+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:43:55.629798+0000","last_undegraded":"2023-03-25T04:43:55.629798+0000","last_fullsized":"2023-03-25T04:43:55.629798+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":42,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:04.389904+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:44:04.389904+0000","last_peered":"2023-03-25T04:44:04.389904+0000","last_clean":"2023-03-25T04:44:04.389904+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:44:04.389904+0000","last_undegraded":"2023-03-25T04:44:04.389904+0000","last_fullsized":"2023-03-25T04:44:04.389904+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:04.389923+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:44:04.389923+0000","last_peered":"2023-03-25T04:44:04.389923+0000","last_clean":"2023-03-25T04:44:04.389923+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:44:04.389923+0000","last_undegraded":"2023-03-25T04:44:04.389923+0000","last_fullsized":"2023-03-25T04:44:04.389923+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117242,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:47:05 2023","interfaces":[{"interface":"back","average":{"1min":0.27800000000000002,"5min":0.27000000000000002,"15min":0.26300000000000001},"min":{"1min":0.20100000000000001,"5min":0.16900000000000001,"15min":0.16},"max":{"1min":0.39200000000000002,"5min":0.42999999999999999,"15min":0.45600000000000002},"last":0.40500000000000003},{"interface":"front","average":{"1min":0.28399999999999997,"5min":0.28100000000000003,"15min":0.27200000000000002},"min":{"1min":0.17799999999999999,"5min":0.17799999999999999,"15min":0.17699999999999999},"max":{"1min":0.42599999999999999,"5min":0.47199999999999998,"15min":0.47199999999999998},"last":0.42399999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051834,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:47:01 2023","interfaces":[{"interface":"back","average":{"1min":0.27300000000000002,"5min":0.26100000000000001,"15min":0.27000000000000002},"min":{"1min":0.186,"5min":0.17399999999999999,"15min":0.17000000000000001},"max":{"1min":0.40400000000000003,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.25600000000000001},{"interface":"front","average":{"1min":0.28699999999999998,"5min":0.27500000000000002,"15min":0.28299999999999997},"min":{"1min":0.186,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.48599999999999999},"last":0.22800000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:47:49.510 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace:     ctx.managers[config['cluster']].wait_for_clean()
,   File "/home/teuthworker/src/github.com_ceph_ceph-c_511015c23eef0684a64171b710e432448df2b7c7/qa/tasks/ceph_manager.py", line 2747, in wait_for_clean
,     assert time.time() - start < timeout, \

Ignoring malparsed backtrace:  ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
,  1: /lib64/libpthread.so.0(+0x12cf0) [0x7f1453c2fcf0]
,  2: __pthread_mutex_lock()

Ignoring malparsed backtrace: 2023-03-25T04:28:05.172 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":142,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":5,"modified":"2023-03-25T04:27:12.092098+0000","services":{}},"progress_events":{}}
, 2023-03-25T04:28:05.185 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now
, 2023-03-25T04:28:05.186 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace: 2023-03-25T04:28:28.360 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":116,"stamp":"2023-03-25T04:28:28.108007+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002578"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.314661+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:27:45.314661+0000","last_peered":"2023-03-25T04:27:45.314661+0000","last_clean":"2023-03-25T04:27:45.314661+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:27:45.314661+0000","last_undegraded":"2023-03-25T04:27:45.314661+0000","last_fullsized":"2023-03-25T04:27:45.314661+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":25,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.310745+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:27:44.310745+0000","last_peered":"2023-03-25T04:27:44.310745+0000","last_clean":"2023-03-25T04:27:44.310745+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:27:44.310745+0000","last_undegraded":"2023-03-25T04:27:44.310745+0000","last_fullsized":"2023-03-25T04:27:44.310745+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":25,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.310650+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:27:44.310650+0000","last_peered":"2023-03-25T04:27:44.310650+0000","last_clean":"2023-03-25T04:27:44.310650+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:27:44.310650+0000","last_undegraded":"2023-03-25T04:27:44.310650+0000","last_fullsized":"2023-03-25T04:27:44.310650+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":26,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.312962+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:27:44.312962+0000","last_peered":"2023-03-25T04:27:44.312962+0000","last_clean":"2023-03-25T04:27:44.312962+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:27:44.312962+0000","last_undegraded":"2023-03-25T04:27:44.312962+0000","last_fullsized":"2023-03-25T04:27:44.312962+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315064+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:27:45.315064+0000","last_peered":"2023-03-25T04:27:45.315064+0000","last_clean":"2023-03-25T04:27:45.315064+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:27:45.315064+0000","last_undegraded":"2023-03-25T04:27:45.315064+0000","last_fullsized":"2023-03-25T04:27:45.315064+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":25,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.310707+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:27:44.310707+0000","last_peered":"2023-03-25T04:27:44.310707+0000","last_clean":"2023-03-25T04:27:44.310707+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:27:44.310707+0000","last_undegraded":"2023-03-25T04:27:44.310707+0000","last_fullsized":"2023-03-25T04:27:44.310707+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117007,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.375}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051597,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.186}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:28:28.362 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean
, 2023-03-25T04:28:28.362 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:28:45.655 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":124,"stamp":"2023-03-25T04:28:44.111265+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002492"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.314661+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:27:45.314661+0000","last_peered":"2023-03-25T04:27:45.314661+0000","last_clean":"2023-03-25T04:27:45.314661+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:27:45.314661+0000","last_undegraded":"2023-03-25T04:27:45.314661+0000","last_fullsized":"2023-03-25T04:27:45.314661+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":25,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.310745+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:27:44.310745+0000","last_peered":"2023-03-25T04:27:44.310745+0000","last_clean":"2023-03-25T04:27:44.310745+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:27:44.310745+0000","last_undegraded":"2023-03-25T04:27:44.310745+0000","last_fullsized":"2023-03-25T04:27:44.310745+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":25,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.310650+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:27:44.310650+0000","last_peered":"2023-03-25T04:27:44.310650+0000","last_clean":"2023-03-25T04:27:44.310650+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:27:44.310650+0000","last_undegraded":"2023-03-25T04:27:44.310650+0000","last_fullsized":"2023-03-25T04:27:44.310650+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":26,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.312962+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:27:44.312962+0000","last_peered":"2023-03-25T04:27:44.312962+0000","last_clean":"2023-03-25T04:27:44.312962+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:27:44.312962+0000","last_undegraded":"2023-03-25T04:27:44.312962+0000","last_fullsized":"2023-03-25T04:27:44.312962+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315064+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:27:45.315064+0000","last_peered":"2023-03-25T04:27:45.315064+0000","last_clean":"2023-03-25T04:27:45.315064+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:27:45.315064+0000","last_undegraded":"2023-03-25T04:27:45.315064+0000","last_fullsized":"2023-03-25T04:27:45.315064+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":25,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.310707+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:27:44.310707+0000","last_peered":"2023-03-25T04:27:44.310707+0000","last_clean":"2023-03-25T04:27:44.310707+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:27:44.310707+0000","last_undegraded":"2023-03-25T04:27:44.310707+0000","last_fullsized":"2023-03-25T04:27:44.310707+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117011,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:28:31 2023","interfaces":[{"interface":"back","average":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.36899999999999999},{"interface":"front","average":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"min":{"1min":0.19,"5min":0.19,"15min":0.19},"max":{"1min":0.46300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.39800000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051601,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:28:24 2023","interfaces":[{"interface":"back","average":{"1min":0.249,"5min":0.249,"15min":0.249},"min":{"1min":0.17699999999999999,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"last":0.254},{"interface":"front","average":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"min":{"1min":0.20200000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"last":0.312}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:28:45.656 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace: 2023-03-25T04:29:05.171 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}
, 2023-03-25T04:29:05.171 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json
, 2023-03-25T04:29:05.561 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:29:22.465 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:29:22.773 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all

Ignoring malparsed backtrace: 2023-03-25T04:29:42.405 INFO:teuthology.orchestra.run.smithi070.stdout:
, 2023-03-25T04:29:42.406 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":240,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":9,"modified":"2023-03-25T04:29:32.120358+0000","services":{"mon":{"daemons":{"summary":"","a":{"start_epoch":0,"start_stamp":"0.000000","gid":0,"addr":"(unrecognized address family 0)/0","metadata":{},"task_status":{}}}}}},"progress_events":{}}
, 2023-03-25T04:29:42.419 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:29:59.627 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":161,"stamp":"2023-03-25T04:29:58.125186+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002398"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315468+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:27:45.315468+0000","last_peered":"2023-03-25T04:27:45.315468+0000","last_clean":"2023-03-25T04:27:45.315468+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:27:45.315468+0000","last_undegraded":"2023-03-25T04:27:45.315468+0000","last_fullsized":"2023-03-25T04:27:45.315468+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312845+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:27:45.312845+0000","last_peered":"2023-03-25T04:27:45.312845+0000","last_clean":"2023-03-25T04:27:45.312845+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:27:45.312845+0000","last_undegraded":"2023-03-25T04:27:45.312845+0000","last_fullsized":"2023-03-25T04:27:45.312845+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312882+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:27:45.312882+0000","last_peered":"2023-03-25T04:27:45.312882+0000","last_clean":"2023-03-25T04:27:45.312882+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:27:45.312882+0000","last_undegraded":"2023-03-25T04:27:45.312882+0000","last_fullsized":"2023-03-25T04:27:45.312882+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.311730+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:27:45.311730+0000","last_peered":"2023-03-25T04:27:45.311730+0000","last_clean":"2023-03-25T04:27:45.311730+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:27:45.311730+0000","last_undegraded":"2023-03-25T04:27:45.311730+0000","last_fullsized":"2023-03-25T04:27:45.311730+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315735+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:27:45.315735+0000","last_peered":"2023-03-25T04:27:45.315735+0000","last_clean":"2023-03-25T04:27:45.315735+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:27:45.315735+0000","last_undegraded":"2023-03-25T04:27:45.315735+0000","last_fullsized":"2023-03-25T04:27:45.315735+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312833+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:27:45.312833+0000","last_peered":"2023-03-25T04:27:45.312833+0000","last_clean":"2023-03-25T04:27:45.312833+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:27:45.312833+0000","last_undegraded":"2023-03-25T04:27:45.312833+0000","last_fullsized":"2023-03-25T04:27:45.312833+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117026,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:29:33 2023","interfaces":[{"interface":"back","average":{"1min":0.26300000000000001,"5min":0.29099999999999998,"15min":0.29599999999999999},"min":{"1min":0.191,"5min":0.191,"15min":0.191},"max":{"1min":0.371,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.20499999999999999},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.29599999999999999,"15min":0.29999999999999999},"min":{"1min":0.21099999999999999,"5min":0.19,"15min":0.19},"max":{"1min":0.39800000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.19500000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051616,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:29:29 2023","interfaces":[{"interface":"back","average":{"1min":0.27200000000000002,"5min":0.253,"15min":0.25},"min":{"1min":0.183,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"last":0.26400000000000001},{"interface":"front","average":{"1min":0.28499999999999998,"5min":0.26800000000000002,"15min":0.26500000000000001},"min":{"1min":0.16600000000000001,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"last":0.29899999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:29:59.629 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean

Ignoring malparsed backtrace: 2023-03-25T04:30:16.889 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:30:16.903 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":170,"stamp":"2023-03-25T04:30:16.128427+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002222"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315468+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:27:45.315468+0000","last_peered":"2023-03-25T04:27:45.315468+0000","last_clean":"2023-03-25T04:27:45.315468+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:27:45.315468+0000","last_undegraded":"2023-03-25T04:27:45.315468+0000","last_fullsized":"2023-03-25T04:27:45.315468+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312845+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:27:45.312845+0000","last_peered":"2023-03-25T04:27:45.312845+0000","last_clean":"2023-03-25T04:27:45.312845+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:27:45.312845+0000","last_undegraded":"2023-03-25T04:27:45.312845+0000","last_fullsized":"2023-03-25T04:27:45.312845+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312882+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:27:45.312882+0000","last_peered":"2023-03-25T04:27:45.312882+0000","last_clean":"2023-03-25T04:27:45.312882+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:27:45.312882+0000","last_undegraded":"2023-03-25T04:27:45.312882+0000","last_fullsized":"2023-03-25T04:27:45.312882+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.311730+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:27:45.311730+0000","last_peered":"2023-03-25T04:27:45.311730+0000","last_clean":"2023-03-25T04:27:45.311730+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:27:45.311730+0000","last_undegraded":"2023-03-25T04:27:45.311730+0000","last_fullsized":"2023-03-25T04:27:45.311730+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315735+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:27:45.315735+0000","last_peered":"2023-03-25T04:27:45.315735+0000","last_clean":"2023-03-25T04:27:45.315735+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:27:45.315735+0000","last_undegraded":"2023-03-25T04:27:45.315735+0000","last_fullsized":"2023-03-25T04:27:45.315735+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312833+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:27:45.312833+0000","last_peered":"2023-03-25T04:27:45.312833+0000","last_clean":"2023-03-25T04:27:45.312833+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:27:45.312833+0000","last_undegraded":"2023-03-25T04:27:45.312833+0000","last_fullsized":"2023-03-25T04:27:45.312833+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117030,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:29:33 2023","interfaces":[{"interface":"back","average":{"1min":0.26300000000000001,"5min":0.29099999999999998,"15min":0.29599999999999999},"min":{"1min":0.191,"5min":0.191,"15min":0.191},"max":{"1min":0.371,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.245},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.29599999999999999,"15min":0.29999999999999999},"min":{"1min":0.21099999999999999,"5min":0.19,"15min":0.19},"max":{"1min":0.39800000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.20899999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051620,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:29:29 2023","interfaces":[{"interface":"back","average":{"1min":0.27200000000000002,"5min":0.253,"15min":0.25},"min":{"1min":0.183,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"last":0.19400000000000001},{"interface":"front","average":{"1min":0.28499999999999998,"5min":0.26800000000000002,"15min":0.26500000000000001},"min":{"1min":0.16600000000000001,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"last":0.23599999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:30:31.984+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:30:32.201+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:30:32.981+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)

Ignoring malparsed backtrace: 2023-03-25T04:30:48.449 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now
, 2023-03-25T04:30:48.450 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:30:48.760 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:31:08.342 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":196,"stamp":"2023-03-25T04:31:08.139018+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002431"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315468+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:27:45.315468+0000","last_peered":"2023-03-25T04:27:45.315468+0000","last_clean":"2023-03-25T04:27:45.315468+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:27:45.315468+0000","last_undegraded":"2023-03-25T04:27:45.315468+0000","last_fullsized":"2023-03-25T04:27:45.315468+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312845+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:27:45.312845+0000","last_peered":"2023-03-25T04:27:45.312845+0000","last_clean":"2023-03-25T04:27:45.312845+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:27:45.312845+0000","last_undegraded":"2023-03-25T04:27:45.312845+0000","last_fullsized":"2023-03-25T04:27:45.312845+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312882+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:27:45.312882+0000","last_peered":"2023-03-25T04:27:45.312882+0000","last_clean":"2023-03-25T04:27:45.312882+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:27:45.312882+0000","last_undegraded":"2023-03-25T04:27:45.312882+0000","last_fullsized":"2023-03-25T04:27:45.312882+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.311730+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:27:45.311730+0000","last_peered":"2023-03-25T04:27:45.311730+0000","last_clean":"2023-03-25T04:27:45.311730+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:27:45.311730+0000","last_undegraded":"2023-03-25T04:27:45.311730+0000","last_fullsized":"2023-03-25T04:27:45.311730+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315735+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:27:45.315735+0000","last_peered":"2023-03-25T04:27:45.315735+0000","last_clean":"2023-03-25T04:27:45.315735+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:27:45.315735+0000","last_undegraded":"2023-03-25T04:27:45.315735+0000","last_fullsized":"2023-03-25T04:27:45.315735+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312833+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:27:45.312833+0000","last_peered":"2023-03-25T04:27:45.312833+0000","last_clean":"2023-03-25T04:27:45.312833+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:27:45.312833+0000","last_undegraded":"2023-03-25T04:27:45.312833+0000","last_fullsized":"2023-03-25T04:27:45.312833+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117040,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:30:37 2023","interfaces":[{"interface":"back","average":{"1min":0.28799999999999998,"5min":0.28899999999999998,"15min":0.29499999999999998},"min":{"1min":0.20200000000000001,"5min":0.191,"15min":0.191},"max":{"1min":0.433,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.29799999999999999},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.28999999999999998,"15min":0.29799999999999999},"min":{"1min":0.17499999999999999,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":0.40300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.25700000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051630,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:30:32 2023","interfaces":[{"interface":"back","average":{"1min":0.25600000000000001,"5min":0.255,"15min":0.251},"min":{"1min":0.19400000000000001,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.35699999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"last":0.22800000000000001},{"interface":"front","average":{"1min":0.28699999999999998,"5min":0.27200000000000002,"15min":0.26600000000000001},"min":{"1min":0.20599999999999999,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.39400000000000002,"5min":0.42899999999999999,"15min":0.42899999999999999},"last":0.24099999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:31:08.343 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace: 2023-03-25T04:31:24.283+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:31:25.192+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:31:25.295+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)

Ignoring malparsed backtrace: 2023-03-25T04:31:40.257 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:31:40.271 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":212,"stamp":"2023-03-25T04:31:40.144732+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002208"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315468+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:27:45.315468+0000","last_peered":"2023-03-25T04:27:45.315468+0000","last_clean":"2023-03-25T04:27:45.315468+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:27:45.315468+0000","last_undegraded":"2023-03-25T04:27:45.315468+0000","last_fullsized":"2023-03-25T04:27:45.315468+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312845+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:27:45.312845+0000","last_peered":"2023-03-25T04:27:45.312845+0000","last_clean":"2023-03-25T04:27:45.312845+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:27:45.312845+0000","last_undegraded":"2023-03-25T04:27:45.312845+0000","last_fullsized":"2023-03-25T04:27:45.312845+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:23.308607+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:29:23.308607+0000","last_peered":"2023-03-25T04:29:23.308607+0000","last_clean":"2023-03-25T04:29:23.308607+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:29:23.308607+0000","last_undegraded":"2023-03-25T04:29:23.308607+0000","last_fullsized":"2023-03-25T04:29:23.308607+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":28,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:20.554710+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:29:20.554710+0000","last_peered":"2023-03-25T04:29:20.554710+0000","last_clean":"2023-03-25T04:29:20.554710+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:29:20.554710+0000","last_undegraded":"2023-03-25T04:29:20.554710+0000","last_fullsized":"2023-03-25T04:29:20.554710+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309216+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:29:28.309216+0000","last_peered":"2023-03-25T04:29:28.309216+0000","last_clean":"2023-03-25T04:29:28.309216+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:29:28.309216+0000","last_undegraded":"2023-03-25T04:29:28.309216+0000","last_fullsized":"2023-03-25T04:29:28.309216+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309233+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:29:28.309233+0000","last_peered":"2023-03-25T04:29:28.309233+0000","last_clean":"2023-03-25T04:29:28.309233+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:29:28.309233+0000","last_undegraded":"2023-03-25T04:29:28.309233+0000","last_fullsized":"2023-03-25T04:29:28.309233+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117046,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:30:37 2023","interfaces":[{"interface":"back","average":{"1min":0.28799999999999998,"5min":0.28899999999999998,"15min":0.29499999999999998},"min":{"1min":0.20200000000000001,"5min":0.191,"15min":0.191},"max":{"1min":0.433,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.216},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.28999999999999998,"15min":0.29799999999999999},"min":{"1min":0.17499999999999999,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":0.40300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.24299999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051637,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:31:35 2023","interfaces":[{"interface":"back","average":{"1min":0.26100000000000001,"5min":0.25700000000000001,"15min":0.251},"min":{"1min":0.19,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.35899999999999999,"5min":0.41599999999999998,"15min":0.41599999999999998},"last":0.221},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.27400000000000002,"15min":0.26700000000000002},"min":{"1min":0.20200000000000001,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.374,"5min":0.42899999999999999,"15min":0.42899999999999999},"last":0.24099999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:32:01.906+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:32:02.388+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:32:02.794 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:32:22.326 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":233,"stamp":"2023-03-25T04:32:22.153289+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002545"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:33.309748+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:29:33.309748+0000","last_peered":"2023-03-25T04:29:33.309748+0000","last_clean":"2023-03-25T04:29:33.309748+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:29:33.309748+0000","last_undegraded":"2023-03-25T04:29:33.309748+0000","last_fullsized":"2023-03-25T04:29:33.309748+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:33.309729+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:29:33.309729+0000","last_peered":"2023-03-25T04:29:33.309729+0000","last_clean":"2023-03-25T04:29:33.309729+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:29:33.309729+0000","last_undegraded":"2023-03-25T04:29:33.309729+0000","last_fullsized":"2023-03-25T04:29:33.309729+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:23.308607+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:29:23.308607+0000","last_peered":"2023-03-25T04:29:23.308607+0000","last_clean":"2023-03-25T04:29:23.308607+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:29:23.308607+0000","last_undegraded":"2023-03-25T04:29:23.308607+0000","last_fullsized":"2023-03-25T04:29:23.308607+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":28,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:20.554710+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:29:20.554710+0000","last_peered":"2023-03-25T04:29:20.554710+0000","last_clean":"2023-03-25T04:29:20.554710+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:29:20.554710+0000","last_undegraded":"2023-03-25T04:29:20.554710+0000","last_fullsized":"2023-03-25T04:29:20.554710+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309216+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:29:28.309216+0000","last_peered":"2023-03-25T04:29:28.309216+0000","last_clean":"2023-03-25T04:29:28.309216+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:29:28.309216+0000","last_undegraded":"2023-03-25T04:29:28.309216+0000","last_fullsized":"2023-03-25T04:29:28.309216+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309233+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:29:28.309233+0000","last_peered":"2023-03-25T04:29:28.309233+0000","last_clean":"2023-03-25T04:29:28.309233+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:29:28.309233+0000","last_undegraded":"2023-03-25T04:29:28.309233+0000","last_fullsized":"2023-03-25T04:29:28.309233+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117055,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:31:42 2023","interfaces":[{"interface":"back","average":{"1min":0.27500000000000002,"5min":0.28399999999999997,"15min":0.29399999999999998},"min":{"1min":0.19800000000000001,"5min":0.191,"15min":0.191},"max":{"1min":0.36899999999999999,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.32300000000000001},{"interface":"front","average":{"1min":0.27900000000000003,"5min":0.28499999999999998,"15min":0.29599999999999999},"min":{"1min":0.216,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":0.40799999999999997,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.27100000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051646,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:31:35 2023","interfaces":[{"interface":"back","average":{"1min":0.26100000000000001,"5min":0.25700000000000001,"15min":0.251},"min":{"1min":0.19,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.35899999999999999,"5min":0.41599999999999998,"15min":0.41599999999999998},"last":0.25600000000000001},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.27400000000000002,"15min":0.26700000000000002},"min":{"1min":0.20200000000000001,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.374,"5min":0.42899999999999999,"15min":0.42899999999999999},"last":0.29199999999999998}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:32:22.327 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean
, 2023-03-25T04:32:22.327 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}

Ignoring malparsed backtrace: 2023-03-25T04:32:39.836 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:32:39.849 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":241,"stamp":"2023-03-25T04:32:38.156312+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002346"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:33.309748+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:29:33.309748+0000","last_peered":"2023-03-25T04:29:33.309748+0000","last_clean":"2023-03-25T04:29:33.309748+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:29:33.309748+0000","last_undegraded":"2023-03-25T04:29:33.309748+0000","last_fullsized":"2023-03-25T04:29:33.309748+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:33.309729+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:29:33.309729+0000","last_peered":"2023-03-25T04:29:33.309729+0000","last_clean":"2023-03-25T04:29:33.309729+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:29:33.309729+0000","last_undegraded":"2023-03-25T04:29:33.309729+0000","last_fullsized":"2023-03-25T04:29:33.309729+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:23.308607+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:29:23.308607+0000","last_peered":"2023-03-25T04:29:23.308607+0000","last_clean":"2023-03-25T04:29:23.308607+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:29:23.308607+0000","last_undegraded":"2023-03-25T04:29:23.308607+0000","last_fullsized":"2023-03-25T04:29:23.308607+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":28,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:20.554710+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:29:20.554710+0000","last_peered":"2023-03-25T04:29:20.554710+0000","last_clean":"2023-03-25T04:29:20.554710+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:29:20.554710+0000","last_undegraded":"2023-03-25T04:29:20.554710+0000","last_fullsized":"2023-03-25T04:29:20.554710+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309216+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:29:28.309216+0000","last_peered":"2023-03-25T04:29:28.309216+0000","last_clean":"2023-03-25T04:29:28.309216+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:29:28.309216+0000","last_undegraded":"2023-03-25T04:29:28.309216+0000","last_fullsized":"2023-03-25T04:29:28.309216+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309233+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:29:28.309233+0000","last_peered":"2023-03-25T04:29:28.309233+0000","last_clean":"2023-03-25T04:29:28.309233+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:29:28.309233+0000","last_undegraded":"2023-03-25T04:29:28.309233+0000","last_fullsized":"2023-03-25T04:29:28.309233+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117058,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:31:42 2023","interfaces":[{"interface":"back","average":{"1min":0.27500000000000002,"5min":0.28399999999999997,"15min":0.29399999999999998},"min":{"1min":0.19800000000000001,"5min":0.191,"15min":0.191},"max":{"1min":0.36899999999999999,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.30599999999999999},{"interface":"front","average":{"1min":0.27900000000000003,"5min":0.28499999999999998,"15min":0.29599999999999999},"min":{"1min":0.216,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":0.40799999999999997,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.32500000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051649,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:31:35 2023","interfaces":[{"interface":"back","average":{"1min":0.26100000000000001,"5min":0.25700000000000001,"15min":0.251},"min":{"1min":0.19,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.35899999999999999,"5min":0.41599999999999998,"15min":0.41599999999999998},"last":0.24199999999999999},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.27400000000000002,"15min":0.26700000000000002},"min":{"1min":0.20200000000000001,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.374,"5min":0.42899999999999999,"15min":0.42899999999999999},"last":0.22600000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:32:54.944+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:32:55.469+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:32:55.927+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)

Ignoring malparsed backtrace: 2023-03-25T04:33:13.958 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean
, 2023-03-25T04:33:13.959 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}
, 2023-03-25T04:33:13.959 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json

Ignoring malparsed backtrace: 2023-03-25T04:33:31.199 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":267,"stamp":"2023-03-25T04:33:30.166660+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002269"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:33.309748+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:29:33.309748+0000","last_peered":"2023-03-25T04:29:33.309748+0000","last_clean":"2023-03-25T04:29:33.309748+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:29:33.309748+0000","last_undegraded":"2023-03-25T04:29:33.309748+0000","last_fullsized":"2023-03-25T04:29:33.309748+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:33.309729+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:29:33.309729+0000","last_peered":"2023-03-25T04:29:33.309729+0000","last_clean":"2023-03-25T04:29:33.309729+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:29:33.309729+0000","last_undegraded":"2023-03-25T04:29:33.309729+0000","last_fullsized":"2023-03-25T04:29:33.309729+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":28,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:31:23.320894+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:31:23.320894+0000","last_peered":"2023-03-25T04:31:23.320894+0000","last_clean":"2023-03-25T04:31:23.320894+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:31:23.320894+0000","last_undegraded":"2023-03-25T04:31:23.320894+0000","last_fullsized":"2023-03-25T04:31:23.320894+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":28,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:20.554710+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:29:20.554710+0000","last_peered":"2023-03-25T04:29:20.554710+0000","last_clean":"2023-03-25T04:29:20.554710+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:29:20.554710+0000","last_undegraded":"2023-03-25T04:29:20.554710+0000","last_fullsized":"2023-03-25T04:29:20.554710+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309216+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:29:28.309216+0000","last_peered":"2023-03-25T04:29:28.309216+0000","last_clean":"2023-03-25T04:29:28.309216+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:29:28.309216+0000","last_undegraded":"2023-03-25T04:29:28.309216+0000","last_fullsized":"2023-03-25T04:29:28.309216+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309233+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:29:28.309233+0000","last_peered":"2023-03-25T04:29:28.309233+0000","last_clean":"2023-03-25T04:29:28.309233+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:29:28.309233+0000","last_undegraded":"2023-03-25T04:29:28.309233+0000","last_fullsized":"2023-03-25T04:29:28.309233+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117068,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:32:46 2023","interfaces":[{"interface":"back","average":{"1min":0.27000000000000002,"5min":0.27900000000000003,"15min":0.29199999999999998},"min":{"1min":0.17899999999999999,"5min":0.17899999999999999,"15min":0.17899999999999999},"max":{"1min":0.35599999999999998,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.28799999999999998},{"interface":"front","average":{"1min":0.26800000000000002,"5min":0.27800000000000002,"15min":0.29399999999999998},"min":{"1min":0.19900000000000001,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":0.377,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.315}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051660,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:32:36 2023","interfaces":[{"interface":"back","average":{"1min":0.248,"5min":0.25700000000000001,"15min":0.251},"min":{"1min":0.17000000000000001,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"last":0.22700000000000001},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.27600000000000002,"15min":0.26800000000000002},"min":{"1min":0.192,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"last":0.20399999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:33:31.201 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:33:31.400+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)

Ignoring malparsed backtrace: 2023-03-25T04:33:47.531+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:33:47.951+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:33:48.505+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)

Ignoring malparsed backtrace: 2023-03-25T04:34:06.104 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json
, 2023-03-25T04:34:06.118+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:34:06.497 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:34:23.399 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:34:23.665+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:34:23.712 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:34:40.195+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:34:40.523+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:34:40.676 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace: 2023-03-25T04:35:03.381+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:35:04.139+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:35:04.391+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)

Ignoring malparsed backtrace: 2023-03-25T04:35:22.738 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean
, 2023-03-25T04:35:22.739 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}
, 2023-03-25T04:35:22.739 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json

Ignoring malparsed backtrace: 2023-03-25T04:35:41.393+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:35:42.352+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:35:42.376+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:35:57.384 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":340,"stamp":"2023-03-25T04:35:56.194992+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.001930"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":36,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:31:43.322894+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:31:43.322894+0000","last_peered":"2023-03-25T04:31:43.322894+0000","last_clean":"2023-03-25T04:31:43.322894+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:31:43.322894+0000","last_undegraded":"2023-03-25T04:31:43.322894+0000","last_fullsized":"2023-03-25T04:31:43.322894+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":28,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:31:43.322876+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:31:43.322876+0000","last_peered":"2023-03-25T04:31:43.322876+0000","last_clean":"2023-03-25T04:31:43.322876+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:31:43.322876+0000","last_undegraded":"2023-03-25T04:31:43.322876+0000","last_fullsized":"2023-03-25T04:31:43.322876+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:23.331998+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:33:23.331998+0000","last_peered":"2023-03-25T04:33:23.331998+0000","last_clean":"2023-03-25T04:33:23.331998+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:33:23.331998+0000","last_undegraded":"2023-03-25T04:33:23.331998+0000","last_fullsized":"2023-03-25T04:33:23.331998+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:30.576723+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:33:30.576723+0000","last_peered":"2023-03-25T04:33:30.576723+0000","last_clean":"2023-03-25T04:33:30.576723+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:33:30.576723+0000","last_undegraded":"2023-03-25T04:33:30.576723+0000","last_fullsized":"2023-03-25T04:33:30.576723+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":37,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:38.333447+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:33:38.333447+0000","last_peered":"2023-03-25T04:33:38.333447+0000","last_clean":"2023-03-25T04:33:38.333447+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:33:38.333447+0000","last_undegraded":"2023-03-25T04:33:38.333447+0000","last_fullsized":"2023-03-25T04:33:38.333447+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:38.333467+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:33:38.333467+0000","last_peered":"2023-03-25T04:33:38.333467+0000","last_clean":"2023-03-25T04:33:38.333467+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:33:38.333467+0000","last_undegraded":"2023-03-25T04:33:38.333467+0000","last_fullsized":"2023-03-25T04:33:38.333467+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117098,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:35:49 2023","interfaces":[{"interface":"back","average":{"1min":0.255,"5min":0.26000000000000001,"15min":0.28199999999999997},"min":{"1min":0.17899999999999999,"5min":0.16,"15min":0.16},"max":{"1min":0.36399999999999999,"5min":0.42599999999999999,"15min":0.45100000000000001},"last":0.26100000000000001},{"interface":"front","average":{"1min":0.26000000000000001,"5min":0.26700000000000002,"15min":0.28599999999999998},"min":{"1min":0.188,"5min":0.17999999999999999,"15min":0.17499999999999999},"max":{"1min":0.39100000000000001,"5min":0.45200000000000001,"15min":0.46300000000000002},"last":0.28399999999999997}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051689,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:35:42 2023","interfaces":[{"interface":"back","average":{"1min":0.29399999999999998,"5min":0.27200000000000002,"15min":0.25800000000000001},"min":{"1min":0.17299999999999999,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.41099999999999998,"5min":0.42699999999999999,"15min":0.42699999999999999},"last":0.20399999999999999},{"interface":"front","average":{"1min":0.30299999999999999,"5min":0.28399999999999997,"15min":0.27300000000000002},"min":{"1min":0.19700000000000001,"5min":0.192,"15min":0.16600000000000001},"max":{"1min":0.43099999999999999,"5min":0.45800000000000002,"15min":0.45800000000000002},"last":0.22}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:35:57.624+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)

Ignoring malparsed backtrace: 2023-03-25T04:36:17.226 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:36:17.239 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":350,"stamp":"2023-03-25T04:36:16.198866+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002148"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":37,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:53.335026+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:33:53.335026+0000","last_peered":"2023-03-25T04:33:53.335026+0000","last_clean":"2023-03-25T04:33:53.335026+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:33:53.335026+0000","last_undegraded":"2023-03-25T04:33:53.335026+0000","last_fullsized":"2023-03-25T04:33:53.335026+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:53.335009+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:33:53.335009+0000","last_peered":"2023-03-25T04:33:53.335009+0000","last_clean":"2023-03-25T04:33:53.335009+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:33:53.335009+0000","last_undegraded":"2023-03-25T04:33:53.335009+0000","last_fullsized":"2023-03-25T04:33:53.335009+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:23.331998+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:33:23.331998+0000","last_peered":"2023-03-25T04:33:23.331998+0000","last_clean":"2023-03-25T04:33:23.331998+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:33:23.331998+0000","last_undegraded":"2023-03-25T04:33:23.331998+0000","last_fullsized":"2023-03-25T04:33:23.331998+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:30.576723+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:33:30.576723+0000","last_peered":"2023-03-25T04:33:30.576723+0000","last_clean":"2023-03-25T04:33:30.576723+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:33:30.576723+0000","last_undegraded":"2023-03-25T04:33:30.576723+0000","last_fullsized":"2023-03-25T04:33:30.576723+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":37,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:38.333447+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:33:38.333447+0000","last_peered":"2023-03-25T04:33:38.333447+0000","last_clean":"2023-03-25T04:33:38.333447+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:33:38.333447+0000","last_undegraded":"2023-03-25T04:33:38.333447+0000","last_fullsized":"2023-03-25T04:33:38.333447+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:38.333467+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:33:38.333467+0000","last_peered":"2023-03-25T04:33:38.333467+0000","last_clean":"2023-03-25T04:33:38.333467+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:33:38.333467+0000","last_undegraded":"2023-03-25T04:33:38.333467+0000","last_fullsized":"2023-03-25T04:33:38.333467+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117102,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:35:49 2023","interfaces":[{"interface":"back","average":{"1min":0.255,"5min":0.26000000000000001,"15min":0.28199999999999997},"min":{"1min":0.17899999999999999,"5min":0.16,"15min":0.16},"max":{"1min":0.36399999999999999,"5min":0.42599999999999999,"15min":0.45100000000000001},"last":0.28399999999999997},{"interface":"front","average":{"1min":0.26000000000000001,"5min":0.26700000000000002,"15min":0.28599999999999998},"min":{"1min":0.188,"5min":0.17999999999999999,"15min":0.17499999999999999},"max":{"1min":0.39100000000000001,"5min":0.45200000000000001,"15min":0.46300000000000002},"last":0.30299999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051693,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:35:42 2023","interfaces":[{"interface":"back","average":{"1min":0.29399999999999998,"5min":0.27200000000000002,"15min":0.25800000000000001},"min":{"1min":0.17299999999999999,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.41099999999999998,"5min":0.42699999999999999,"15min":0.42699999999999999},"last":0.35099999999999998},{"interface":"front","average":{"1min":0.30299999999999999,"5min":0.28399999999999997,"15min":0.27300000000000002},"min":{"1min":0.19700000000000001,"5min":0.192,"15min":0.16600000000000001},"max":{"1min":0.43099999999999999,"5min":0.45800000000000002,"15min":0.45800000000000002},"last":0.32900000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:36:34.556+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:36:34.717+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:36:35.583+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)

Ignoring malparsed backtrace: 2023-03-25T04:36:52.680 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json
, 2023-03-25T04:36:52.701+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:36:53.067 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:37:09.963 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:37:10.273 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all

Ignoring malparsed backtrace: 2023-03-25T04:37:26.856+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:37:27.288 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:37:27.538+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:37:41.903 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":392,"stamp":"2023-03-25T04:37:40.213373+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002012"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":37,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:53.335026+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:33:53.335026+0000","last_peered":"2023-03-25T04:33:53.335026+0000","last_clean":"2023-03-25T04:33:53.335026+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:33:53.335026+0000","last_undegraded":"2023-03-25T04:33:53.335026+0000","last_fullsized":"2023-03-25T04:33:53.335026+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:53.335009+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:33:53.335009+0000","last_peered":"2023-03-25T04:33:53.335009+0000","last_clean":"2023-03-25T04:33:53.335009+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:33:53.335009+0000","last_undegraded":"2023-03-25T04:33:53.335009+0000","last_fullsized":"2023-03-25T04:33:53.335009+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:23.343585+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:35:23.343585+0000","last_peered":"2023-03-25T04:35:23.343585+0000","last_clean":"2023-03-25T04:35:23.343585+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:35:23.343585+0000","last_undegraded":"2023-03-25T04:35:23.343585+0000","last_fullsized":"2023-03-25T04:35:23.343585+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:30.576723+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:33:30.576723+0000","last_peered":"2023-03-25T04:33:30.576723+0000","last_clean":"2023-03-25T04:33:30.576723+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:33:30.576723+0000","last_undegraded":"2023-03-25T04:33:30.576723+0000","last_fullsized":"2023-03-25T04:33:30.576723+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":37,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:38.333447+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:33:38.333447+0000","last_peered":"2023-03-25T04:33:38.333447+0000","last_clean":"2023-03-25T04:33:38.333447+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:33:38.333447+0000","last_undegraded":"2023-03-25T04:33:38.333447+0000","last_fullsized":"2023-03-25T04:33:38.333447+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:38.333467+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:33:38.333467+0000","last_peered":"2023-03-25T04:33:38.333467+0000","last_clean":"2023-03-25T04:33:38.333467+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:33:38.333467+0000","last_undegraded":"2023-03-25T04:33:38.333467+0000","last_fullsized":"2023-03-25T04:33:38.333467+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117119,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:36:50 2023","interfaces":[{"interface":"back","average":{"1min":0.254,"5min":0.25600000000000001,"15min":0.27900000000000003},"min":{"1min":0.20499999999999999,"5min":0.16,"15min":0.16},"max":{"1min":0.371,"5min":0.42599999999999999,"15min":0.45100000000000001},"last":0.23000000000000001},{"interface":"front","average":{"1min":0.26800000000000002,"5min":0.26500000000000001,"15min":0.28399999999999997},"min":{"1min":0.19700000000000001,"5min":0.17999999999999999,"15min":0.17499999999999999},"max":{"1min":0.34699999999999998,"5min":0.45200000000000001,"15min":0.46300000000000002},"last":0.20899999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051710,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:36:44 2023","interfaces":[{"interface":"back","average":{"1min":0.28399999999999997,"5min":0.27700000000000002,"15min":0.26100000000000001},"min":{"1min":0.192,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.40899999999999997,"5min":0.42699999999999999,"15min":0.42699999999999999},"last":0.39700000000000002},{"interface":"front","average":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.27500000000000002},"min":{"1min":0.182,"5min":0.182,"15min":0.16600000000000001},"max":{"1min":0.439,"5min":0.45800000000000002,"15min":0.45800000000000002},"last":0.439}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:37:42.400+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)

Ignoring malparsed backtrace: 2023-03-25T04:38:04.766+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:38:04.922 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:38:05.246 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:38:19.605 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":411,"stamp":"2023-03-25T04:38:18.220966+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002368"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":38,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:36:03.347070+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:36:03.347070+0000","last_peered":"2023-03-25T04:36:03.347070+0000","last_clean":"2023-03-25T04:36:03.347070+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:36:03.347070+0000","last_undegraded":"2023-03-25T04:36:03.347070+0000","last_fullsized":"2023-03-25T04:36:03.347070+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:36:03.347025+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:36:03.347025+0000","last_peered":"2023-03-25T04:36:03.347025+0000","last_clean":"2023-03-25T04:36:03.347025+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:36:03.347025+0000","last_undegraded":"2023-03-25T04:36:03.347025+0000","last_fullsized":"2023-03-25T04:36:03.347025+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:23.343585+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:35:23.343585+0000","last_peered":"2023-03-25T04:35:23.343585+0000","last_clean":"2023-03-25T04:35:23.343585+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:35:23.343585+0000","last_undegraded":"2023-03-25T04:35:23.343585+0000","last_fullsized":"2023-03-25T04:35:23.343585+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:35.587025+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:35:35.587025+0000","last_peered":"2023-03-25T04:35:35.587025+0000","last_clean":"2023-03-25T04:35:35.587025+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:35:35.587025+0000","last_undegraded":"2023-03-25T04:35:35.587025+0000","last_fullsized":"2023-03-25T04:35:35.587025+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":38,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:43.345308+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:35:43.345308+0000","last_peered":"2023-03-25T04:35:43.345308+0000","last_clean":"2023-03-25T04:35:43.345308+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:35:43.345308+0000","last_undegraded":"2023-03-25T04:35:43.345308+0000","last_fullsized":"2023-03-25T04:35:43.345308+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:43.345324+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:35:43.345324+0000","last_peered":"2023-03-25T04:35:43.345324+0000","last_clean":"2023-03-25T04:35:43.345324+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:35:43.345324+0000","last_undegraded":"2023-03-25T04:35:43.345324+0000","last_fullsized":"2023-03-25T04:35:43.345324+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117127,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:37:53 2023","interfaces":[{"interface":"back","average":{"1min":0.25800000000000001,"5min":0.253,"15min":0.27700000000000002},"min":{"1min":0.184,"5min":0.16,"15min":0.16},"max":{"1min":0.44600000000000001,"5min":0.44600000000000001,"15min":0.45100000000000001},"last":0.35199999999999998},{"interface":"front","average":{"1min":0.26900000000000002,"5min":0.26500000000000001,"15min":0.28199999999999997},"min":{"1min":0.192,"5min":0.17999999999999999,"15min":0.17499999999999999},"max":{"1min":0.46400000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"last":0.33200000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051717,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:37:44 2023","interfaces":[{"interface":"back","average":{"1min":0.27300000000000002,"5min":0.28199999999999997,"15min":0.26200000000000001},"min":{"1min":0.17899999999999999,"5min":0.17299999999999999,"15min":0.17000000000000001},"max":{"1min":0.41799999999999998,"5min":0.42099999999999999,"15min":0.42699999999999999},"last":0.28699999999999998},{"interface":"front","average":{"1min":0.29399999999999998,"5min":0.29099999999999998,"15min":0.27700000000000002},"min":{"1min":0.19900000000000001,"5min":0.182,"15min":0.16600000000000001},"max":{"1min":0.441,"5min":0.441,"15min":0.45800000000000002},"last":0.26300000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:38:19.718+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:38:20.315+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)

Ignoring malparsed backtrace: 2023-03-25T04:38:44.283 INFO:teuthology.orchestra.run.smithi070.stdout:
, 2023-03-25T04:38:44.283 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":781,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":16,"modified":"2023-03-25T04:38:40.225321+0000","services":{}},"progress_events":{}}
, 2023-03-25T04:38:44.296 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now

Ignoring malparsed backtrace: 2023-03-25T04:39:02.477+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:39:02.578+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:39:03.524+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:39:20.359 INFO:teuthology.orchestra.run.smithi070.stdout:
, 2023-03-25T04:39:20.359 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":818,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":16,"modified":"2023-03-25T04:38:40.225321+0000","services":{}},"progress_events":{}}
, 2023-03-25T04:39:20.372 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now

Ignoring malparsed backtrace: 2023-03-25T04:39:37.527 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:39:37.541 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":450,"stamp":"2023-03-25T04:39:36.235332+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002086"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":38,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:36:03.347070+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:36:03.347070+0000","last_peered":"2023-03-25T04:36:03.347070+0000","last_clean":"2023-03-25T04:36:03.347070+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:36:03.347070+0000","last_undegraded":"2023-03-25T04:36:03.347070+0000","last_fullsized":"2023-03-25T04:36:03.347070+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:36:03.347025+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:36:03.347025+0000","last_peered":"2023-03-25T04:36:03.347025+0000","last_clean":"2023-03-25T04:36:03.347025+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:36:03.347025+0000","last_undegraded":"2023-03-25T04:36:03.347025+0000","last_fullsized":"2023-03-25T04:36:03.347025+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:23.353778+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:37:23.353778+0000","last_peered":"2023-03-25T04:37:23.353778+0000","last_clean":"2023-03-25T04:37:23.353778+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:37:23.353778+0000","last_undegraded":"2023-03-25T04:37:23.353778+0000","last_fullsized":"2023-03-25T04:37:23.353778+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:35.587025+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:35:35.587025+0000","last_peered":"2023-03-25T04:35:35.587025+0000","last_clean":"2023-03-25T04:35:35.587025+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:35:35.587025+0000","last_undegraded":"2023-03-25T04:35:35.587025+0000","last_fullsized":"2023-03-25T04:35:35.587025+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":38,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:43.345308+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:35:43.345308+0000","last_peered":"2023-03-25T04:35:43.345308+0000","last_clean":"2023-03-25T04:35:43.345308+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:35:43.345308+0000","last_undegraded":"2023-03-25T04:35:43.345308+0000","last_fullsized":"2023-03-25T04:35:43.345308+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:43.345324+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:35:43.345324+0000","last_peered":"2023-03-25T04:35:43.345324+0000","last_clean":"2023-03-25T04:35:43.345324+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:35:43.345324+0000","last_undegraded":"2023-03-25T04:35:43.345324+0000","last_fullsized":"2023-03-25T04:35:43.345324+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117143,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:38:54 2023","interfaces":[{"interface":"back","average":{"1min":0.25800000000000001,"5min":0.25700000000000001,"15min":0.27400000000000002},"min":{"1min":0.192,"5min":0.17399999999999999,"15min":0.16},"max":{"1min":0.39400000000000002,"5min":0.44600000000000001,"15min":0.45100000000000001},"last":0.26500000000000001},{"interface":"front","average":{"1min":0.25900000000000001,"5min":0.26700000000000002,"15min":0.27900000000000003},"min":{"1min":0.19500000000000001,"5min":0.188,"15min":0.17499999999999999},"max":{"1min":0.41499999999999998,"5min":0.46400000000000002,"15min":0.46400000000000002},"last":0.29099999999999998}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051733,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:38:45 2023","interfaces":[{"interface":"back","average":{"1min":0.27300000000000002,"5min":0.28100000000000003,"15min":0.26400000000000001},"min":{"1min":0.19400000000000001,"5min":0.17299999999999999,"15min":0.17000000000000001},"max":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"last":0.27300000000000002},{"interface":"front","average":{"1min":0.28799999999999998,"5min":0.29099999999999998,"15min":0.27800000000000002},"min":{"1min":0.19800000000000001,"5min":0.182,"15min":0.16600000000000001},"max":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"last":0.308}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:39:54.416+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:39:54.455 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:39:54.780 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:40:09.288+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:40:09.389+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:40:10.267+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)

Ignoring malparsed backtrace: 2023-03-25T04:40:28.984 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":476,"stamp":"2023-03-25T04:40:28.245911+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002693"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":39,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:38:13.358968+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:38:13.358968+0000","last_peered":"2023-03-25T04:38:13.358968+0000","last_clean":"2023-03-25T04:38:13.358968+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:38:13.358968+0000","last_undegraded":"2023-03-25T04:38:13.358968+0000","last_fullsized":"2023-03-25T04:38:13.358968+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:38:13.358922+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:38:13.358922+0000","last_peered":"2023-03-25T04:38:13.358922+0000","last_clean":"2023-03-25T04:38:13.358922+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:38:13.358922+0000","last_undegraded":"2023-03-25T04:38:13.358922+0000","last_fullsized":"2023-03-25T04:38:13.358922+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:23.353778+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:37:23.353778+0000","last_peered":"2023-03-25T04:37:23.353778+0000","last_clean":"2023-03-25T04:37:23.353778+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:37:23.353778+0000","last_undegraded":"2023-03-25T04:37:23.353778+0000","last_fullsized":"2023-03-25T04:37:23.353778+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:40.598579+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:37:40.598579+0000","last_peered":"2023-03-25T04:37:40.598579+0000","last_clean":"2023-03-25T04:37:40.598579+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:37:40.598579+0000","last_undegraded":"2023-03-25T04:37:40.598579+0000","last_fullsized":"2023-03-25T04:37:40.598579+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":39,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:48.356344+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:37:48.356344+0000","last_peered":"2023-03-25T04:37:48.356344+0000","last_clean":"2023-03-25T04:37:48.356344+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:37:48.356344+0000","last_undegraded":"2023-03-25T04:37:48.356344+0000","last_fullsized":"2023-03-25T04:37:48.356344+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:48.356365+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:37:48.356365+0000","last_peered":"2023-03-25T04:37:48.356365+0000","last_clean":"2023-03-25T04:37:48.356365+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:37:48.356365+0000","last_undegraded":"2023-03-25T04:37:48.356365+0000","last_fullsized":"2023-03-25T04:37:48.356365+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117153,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:39:54 2023","interfaces":[{"interface":"back","average":{"1min":0.25900000000000001,"5min":0.25600000000000001,"15min":0.27100000000000002},"min":{"1min":0.188,"5min":0.17899999999999999,"15min":0.16},"max":{"1min":0.41999999999999998,"5min":0.44600000000000001,"15min":0.45100000000000001},"last":0.255},{"interface":"front","average":{"1min":0.27500000000000002,"5min":0.26600000000000001,"15min":0.27700000000000002},"min":{"1min":0.17699999999999999,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.45600000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"last":0.26800000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051743,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:39:47 2023","interfaces":[{"interface":"back","average":{"1min":0.26300000000000001,"5min":0.27700000000000002,"15min":0.26500000000000001},"min":{"1min":0.187,"5min":0.17299999999999999,"15min":0.17000000000000001},"max":{"1min":0.433,"5min":0.433,"15min":0.433},"last":0.36599999999999999},{"interface":"front","average":{"1min":0.27100000000000002,"5min":0.28799999999999998,"15min":0.27900000000000003},"min":{"1min":0.20100000000000001,"5min":0.182,"15min":0.16600000000000001},"max":{"1min":0.48599999999999999,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.34999999999999998}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:40:28.986 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean
, 2023-03-25T04:40:28.986 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}

Ignoring malparsed backtrace: 2023-03-25T04:40:47.229+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:40:47.503+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:40:48.251+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)

Ignoring malparsed backtrace: 2023-03-25T04:41:03.267 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:41:03.578 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all

Ignoring malparsed backtrace: 2023-03-25T04:41:21.347 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":502,"stamp":"2023-03-25T04:41:20.256204+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002342"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":39,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:38:13.358968+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:38:13.358968+0000","last_peered":"2023-03-25T04:38:13.358968+0000","last_clean":"2023-03-25T04:38:13.358968+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:38:13.358968+0000","last_undegraded":"2023-03-25T04:38:13.358968+0000","last_fullsized":"2023-03-25T04:38:13.358968+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:38:13.358922+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:38:13.358922+0000","last_peered":"2023-03-25T04:38:13.358922+0000","last_clean":"2023-03-25T04:38:13.358922+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:38:13.358922+0000","last_undegraded":"2023-03-25T04:38:13.358922+0000","last_fullsized":"2023-03-25T04:38:13.358922+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:23.353778+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:37:23.353778+0000","last_peered":"2023-03-25T04:37:23.353778+0000","last_clean":"2023-03-25T04:37:23.353778+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:37:23.353778+0000","last_undegraded":"2023-03-25T04:37:23.353778+0000","last_fullsized":"2023-03-25T04:37:23.353778+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:40.598579+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:37:40.598579+0000","last_peered":"2023-03-25T04:37:40.598579+0000","last_clean":"2023-03-25T04:37:40.598579+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:37:40.598579+0000","last_undegraded":"2023-03-25T04:37:40.598579+0000","last_fullsized":"2023-03-25T04:37:40.598579+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":39,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:48.356344+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:37:48.356344+0000","last_peered":"2023-03-25T04:37:48.356344+0000","last_clean":"2023-03-25T04:37:48.356344+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:37:48.356344+0000","last_undegraded":"2023-03-25T04:37:48.356344+0000","last_fullsized":"2023-03-25T04:37:48.356344+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:48.356365+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:37:48.356365+0000","last_peered":"2023-03-25T04:37:48.356365+0000","last_clean":"2023-03-25T04:37:48.356365+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:37:48.356365+0000","last_undegraded":"2023-03-25T04:37:48.356365+0000","last_fullsized":"2023-03-25T04:37:48.356365+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117163,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:40:56 2023","interfaces":[{"interface":"back","average":{"1min":0.28799999999999998,"5min":0.26300000000000001,"15min":0.27100000000000002},"min":{"1min":0.188,"5min":0.184,"15min":0.16},"max":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.17499999999999999},{"interface":"front","average":{"1min":0.29599999999999999,"5min":0.27300000000000002,"15min":0.27700000000000002},"min":{"1min":0.20599999999999999,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.43099999999999999,"5min":0.46400000000000002,"15min":0.46400000000000002},"last":0.20699999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051754,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:40:52 2023","interfaces":[{"interface":"back","average":{"1min":0.28599999999999998,"5min":0.27500000000000002,"15min":0.26700000000000002},"min":{"1min":0.17899999999999999,"5min":0.17899999999999999,"15min":0.17000000000000001},"max":{"1min":0.44,"5min":0.44,"15min":0.44},"last":0.23699999999999999},{"interface":"front","average":{"1min":0.29799999999999999,"5min":0.28699999999999998,"15min":0.28100000000000003},"min":{"1min":0.20300000000000001,"5min":0.182,"15min":0.16600000000000001},"max":{"1min":0.47599999999999998,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.28000000000000003}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:41:21.349 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean
, 2023-03-25T04:41:21.349 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}

Ignoring malparsed backtrace: 2023-03-25T04:41:39.867+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:41:40.383+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:41:40.905+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:41:58.219 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json
, 2023-03-25T04:41:58.319+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:41:58.644 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:42:18.193 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":530,"stamp":"2023-03-25T04:42:16.266659+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002356"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":39,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:38:13.358968+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:38:13.358968+0000","last_peered":"2023-03-25T04:38:13.358968+0000","last_clean":"2023-03-25T04:38:13.358968+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:38:13.358968+0000","last_undegraded":"2023-03-25T04:38:13.358968+0000","last_fullsized":"2023-03-25T04:38:13.358968+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:38:13.358922+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:38:13.358922+0000","last_peered":"2023-03-25T04:38:13.358922+0000","last_clean":"2023-03-25T04:38:13.358922+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:38:13.358922+0000","last_undegraded":"2023-03-25T04:38:13.358922+0000","last_fullsized":"2023-03-25T04:38:13.358922+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:23.364593+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:39:23.364593+0000","last_peered":"2023-03-25T04:39:23.364593+0000","last_clean":"2023-03-25T04:39:23.364593+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:39:23.364593+0000","last_undegraded":"2023-03-25T04:39:23.364593+0000","last_fullsized":"2023-03-25T04:39:23.364593+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:45.608475+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:39:45.608475+0000","last_peered":"2023-03-25T04:39:45.608475+0000","last_clean":"2023-03-25T04:39:45.608475+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:39:45.608475+0000","last_undegraded":"2023-03-25T04:39:45.608475+0000","last_fullsized":"2023-03-25T04:39:45.608475+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":40,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:53.367398+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:39:53.367398+0000","last_peered":"2023-03-25T04:39:53.367398+0000","last_clean":"2023-03-25T04:39:53.367398+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:39:53.367398+0000","last_undegraded":"2023-03-25T04:39:53.367398+0000","last_fullsized":"2023-03-25T04:39:53.367398+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:53.367424+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:39:53.367424+0000","last_peered":"2023-03-25T04:39:53.367424+0000","last_clean":"2023-03-25T04:39:53.367424+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:39:53.367424+0000","last_undegraded":"2023-03-25T04:39:53.367424+0000","last_fullsized":"2023-03-25T04:39:53.367424+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117175,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:41:58 2023","interfaces":[{"interface":"back","average":{"1min":0.25800000000000001,"5min":0.26400000000000001,"15min":0.26800000000000002},"min":{"1min":0.17499999999999999,"5min":0.17499999999999999,"15min":0.16},"max":{"1min":0.39100000000000001,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.22900000000000001},{"interface":"front","average":{"1min":0.26300000000000001,"5min":0.27200000000000002,"15min":0.27400000000000002},"min":{"1min":0.20599999999999999,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.35699999999999998,"5min":0.46400000000000002,"15min":0.46400000000000002},"last":0.26400000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051765,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:41:52 2023","interfaces":[{"interface":"back","average":{"1min":0.26500000000000001,"5min":0.27200000000000002,"15min":0.26800000000000002},"min":{"1min":0.17000000000000001,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.39200000000000002,"5min":0.44,"15min":0.44},"last":0.23899999999999999},{"interface":"front","average":{"1min":0.28100000000000003,"5min":0.28599999999999998,"15min":0.28199999999999997},"min":{"1min":0.19400000000000001,"5min":0.19400000000000001,"15min":0.16600000000000001},"max":{"1min":0.41399999999999998,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.254}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:42:18.297+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:42:18.958+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:42:35.512 INFO:teuthology.orchestra.run.smithi070.stdout:
, 2023-03-25T04:42:35.512 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":1013,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":17,"modified":"2023-03-25T04:41:54.262744+0000","services":{}},"progress_events":{}}
, 2023-03-25T04:42:35.525 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:42:52.741 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":548,"stamp":"2023-03-25T04:42:52.273601+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002221"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":40,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:40:23.370056+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:40:23.370056+0000","last_peered":"2023-03-25T04:40:23.370056+0000","last_clean":"2023-03-25T04:40:23.370056+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:40:23.370056+0000","last_undegraded":"2023-03-25T04:40:23.370056+0000","last_fullsized":"2023-03-25T04:40:23.370056+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:40:23.370039+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:40:23.370039+0000","last_peered":"2023-03-25T04:40:23.370039+0000","last_clean":"2023-03-25T04:40:23.370039+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:40:23.370039+0000","last_undegraded":"2023-03-25T04:40:23.370039+0000","last_fullsized":"2023-03-25T04:40:23.370039+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:23.364593+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:39:23.364593+0000","last_peered":"2023-03-25T04:39:23.364593+0000","last_clean":"2023-03-25T04:39:23.364593+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:39:23.364593+0000","last_undegraded":"2023-03-25T04:39:23.364593+0000","last_fullsized":"2023-03-25T04:39:23.364593+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:45.608475+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:39:45.608475+0000","last_peered":"2023-03-25T04:39:45.608475+0000","last_clean":"2023-03-25T04:39:45.608475+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:39:45.608475+0000","last_undegraded":"2023-03-25T04:39:45.608475+0000","last_fullsized":"2023-03-25T04:39:45.608475+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":40,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:53.367398+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:39:53.367398+0000","last_peered":"2023-03-25T04:39:53.367398+0000","last_clean":"2023-03-25T04:39:53.367398+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:39:53.367398+0000","last_undegraded":"2023-03-25T04:39:53.367398+0000","last_fullsized":"2023-03-25T04:39:53.367398+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:53.367424+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:39:53.367424+0000","last_peered":"2023-03-25T04:39:53.367424+0000","last_clean":"2023-03-25T04:39:53.367424+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:39:53.367424+0000","last_undegraded":"2023-03-25T04:39:53.367424+0000","last_fullsized":"2023-03-25T04:39:53.367424+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117182,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:41:58 2023","interfaces":[{"interface":"back","average":{"1min":0.25800000000000001,"5min":0.26400000000000001,"15min":0.26800000000000002},"min":{"1min":0.17499999999999999,"5min":0.17499999999999999,"15min":0.16},"max":{"1min":0.39100000000000001,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.19600000000000001},{"interface":"front","average":{"1min":0.26300000000000001,"5min":0.27200000000000002,"15min":0.27400000000000002},"min":{"1min":0.20599999999999999,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.35699999999999998,"5min":0.46400000000000002,"15min":0.46400000000000002},"last":0.219}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051773,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:41:52 2023","interfaces":[{"interface":"back","average":{"1min":0.26500000000000001,"5min":0.27200000000000002,"15min":0.26800000000000002},"min":{"1min":0.17000000000000001,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.39200000000000002,"5min":0.44,"15min":0.44},"last":0.19},{"interface":"front","average":{"1min":0.28100000000000003,"5min":0.28599999999999998,"15min":0.28199999999999997},"min":{"1min":0.19400000000000001,"5min":0.19400000000000001,"15min":0.16600000000000001},"max":{"1min":0.41399999999999998,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.17599999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:42:52.742 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean

Ignoring malparsed backtrace: 2023-03-25T04:43:09.973 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:43:09.986 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":556,"stamp":"2023-03-25T04:43:08.276556+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002359"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":40,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:40:23.370056+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:40:23.370056+0000","last_peered":"2023-03-25T04:40:23.370056+0000","last_clean":"2023-03-25T04:40:23.370056+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:40:23.370056+0000","last_undegraded":"2023-03-25T04:40:23.370056+0000","last_fullsized":"2023-03-25T04:40:23.370056+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:40:23.370039+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:40:23.370039+0000","last_peered":"2023-03-25T04:40:23.370039+0000","last_clean":"2023-03-25T04:40:23.370039+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:40:23.370039+0000","last_undegraded":"2023-03-25T04:40:23.370039+0000","last_fullsized":"2023-03-25T04:40:23.370039+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:23.364593+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:39:23.364593+0000","last_peered":"2023-03-25T04:39:23.364593+0000","last_clean":"2023-03-25T04:39:23.364593+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:39:23.364593+0000","last_undegraded":"2023-03-25T04:39:23.364593+0000","last_fullsized":"2023-03-25T04:39:23.364593+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:45.608475+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:39:45.608475+0000","last_peered":"2023-03-25T04:39:45.608475+0000","last_clean":"2023-03-25T04:39:45.608475+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:39:45.608475+0000","last_undegraded":"2023-03-25T04:39:45.608475+0000","last_fullsized":"2023-03-25T04:39:45.608475+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":40,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:53.367398+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:39:53.367398+0000","last_peered":"2023-03-25T04:39:53.367398+0000","last_clean":"2023-03-25T04:39:53.367398+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:39:53.367398+0000","last_undegraded":"2023-03-25T04:39:53.367398+0000","last_fullsized":"2023-03-25T04:39:53.367398+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:53.367424+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:39:53.367424+0000","last_peered":"2023-03-25T04:39:53.367424+0000","last_clean":"2023-03-25T04:39:53.367424+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:39:53.367424+0000","last_undegraded":"2023-03-25T04:39:53.367424+0000","last_fullsized":"2023-03-25T04:39:53.367424+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117185,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:43:00 2023","interfaces":[{"interface":"back","average":{"1min":0.27400000000000002,"5min":0.26700000000000002,"15min":0.26600000000000001},"min":{"1min":0.19600000000000001,"5min":0.17499999999999999,"15min":0.16},"max":{"1min":0.42999999999999999,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.27800000000000002},{"interface":"front","average":{"1min":0.28199999999999997,"5min":0.27500000000000002,"15min":0.27300000000000002},"min":{"1min":0.185,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.47199999999999998,"5min":0.47199999999999998,"15min":0.47199999999999998},"last":0.247}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051776,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:42:54 2023","interfaces":[{"interface":"back","average":{"1min":0.26300000000000001,"5min":0.27000000000000002,"15min":0.26900000000000002},"min":{"1min":0.17399999999999999,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.438,"5min":0.44,"15min":0.44},"last":0.20100000000000001},{"interface":"front","average":{"1min":0.27400000000000002,"5min":0.28199999999999997,"15min":0.28299999999999997},"min":{"1min":0.17599999999999999,"5min":0.17599999999999999,"15min":0.16600000000000001},"max":{"1min":0.41599999999999998,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.23499999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:43:25.042+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:43:25.411+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:43:26.013+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:43:47.937+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:43:48.534+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:43:48.889+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:44:09.318 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:44:09.331 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":586,"stamp":"2023-03-25T04:44:08.288933+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002697"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":40,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:40:23.370056+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:40:23.370056+0000","last_peered":"2023-03-25T04:40:23.370056+0000","last_clean":"2023-03-25T04:40:23.370056+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:40:23.370056+0000","last_undegraded":"2023-03-25T04:40:23.370056+0000","last_fullsized":"2023-03-25T04:40:23.370056+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:40:23.370039+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:40:23.370039+0000","last_peered":"2023-03-25T04:40:23.370039+0000","last_clean":"2023-03-25T04:40:23.370039+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:40:23.370039+0000","last_undegraded":"2023-03-25T04:40:23.370039+0000","last_fullsized":"2023-03-25T04:40:23.370039+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:23.375106+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:41:23.375106+0000","last_peered":"2023-03-25T04:41:23.375106+0000","last_clean":"2023-03-25T04:41:23.375106+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:41:23.375106+0000","last_undegraded":"2023-03-25T04:41:23.375106+0000","last_fullsized":"2023-03-25T04:41:23.375106+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:50.618930+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:41:50.618930+0000","last_peered":"2023-03-25T04:41:50.618930+0000","last_clean":"2023-03-25T04:41:50.618930+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:41:50.618930+0000","last_undegraded":"2023-03-25T04:41:50.618930+0000","last_fullsized":"2023-03-25T04:41:50.618930+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":41,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:58.377821+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:41:58.377821+0000","last_peered":"2023-03-25T04:41:58.377821+0000","last_clean":"2023-03-25T04:41:58.377821+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:41:58.377821+0000","last_undegraded":"2023-03-25T04:41:58.377821+0000","last_fullsized":"2023-03-25T04:41:58.377821+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:58.377836+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:41:58.377836+0000","last_peered":"2023-03-25T04:41:58.377836+0000","last_clean":"2023-03-25T04:41:58.377836+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:41:58.377836+0000","last_undegraded":"2023-03-25T04:41:58.377836+0000","last_fullsized":"2023-03-25T04:41:58.377836+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117197,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:44:01 2023","interfaces":[{"interface":"back","average":{"1min":0.27200000000000002,"5min":0.27000000000000002,"15min":0.26400000000000001},"min":{"1min":0.20399999999999999,"5min":0.17499999999999999,"15min":0.16},"max":{"1min":0.38400000000000001,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.245},{"interface":"front","average":{"1min":0.27600000000000002,"5min":0.27800000000000002,"15min":0.27100000000000002},"min":{"1min":0.23100000000000001,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.40400000000000003,"5min":0.47199999999999998,"15min":0.47199999999999998},"last":0.27500000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051788,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:43:55 2023","interfaces":[{"interface":"back","average":{"1min":0.26100000000000001,"5min":0.26700000000000002,"15min":0.27000000000000002},"min":{"1min":0.17799999999999999,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.374,"5min":0.44,"15min":0.44},"last":0.30299999999999999},{"interface":"front","average":{"1min":0.28299999999999997,"5min":0.28100000000000003,"15min":0.28399999999999997},"min":{"1min":0.20899999999999999,"5min":0.17599999999999999,"15min":0.16600000000000001},"max":{"1min":0.40100000000000002,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.27800000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:44:28.923 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":1126,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":19,"modified":"2023-03-25T04:43:20.278959+0000","services":{}},"progress_events":{}}
, 2023-03-25T04:44:28.936 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now
, 2023-03-25T04:44:28.937 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace: 2023-03-25T04:44:48.786 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:44:48.800 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":606,"stamp":"2023-03-25T04:44:48.296805+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002182"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":41,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:42:34.382196+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:42:34.382196+0000","last_peered":"2023-03-25T04:42:34.382196+0000","last_clean":"2023-03-25T04:42:34.382196+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:42:34.382196+0000","last_undegraded":"2023-03-25T04:42:34.382196+0000","last_fullsized":"2023-03-25T04:42:34.382196+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:42:29.381744+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:42:29.381744+0000","last_peered":"2023-03-25T04:42:29.381744+0000","last_clean":"2023-03-25T04:42:29.381744+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:42:29.381744+0000","last_undegraded":"2023-03-25T04:42:29.381744+0000","last_fullsized":"2023-03-25T04:42:29.381744+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:23.375106+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:41:23.375106+0000","last_peered":"2023-03-25T04:41:23.375106+0000","last_clean":"2023-03-25T04:41:23.375106+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:41:23.375106+0000","last_undegraded":"2023-03-25T04:41:23.375106+0000","last_fullsized":"2023-03-25T04:41:23.375106+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:50.618930+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:41:50.618930+0000","last_peered":"2023-03-25T04:41:50.618930+0000","last_clean":"2023-03-25T04:41:50.618930+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:41:50.618930+0000","last_undegraded":"2023-03-25T04:41:50.618930+0000","last_fullsized":"2023-03-25T04:41:50.618930+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":41,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:58.377821+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:41:58.377821+0000","last_peered":"2023-03-25T04:41:58.377821+0000","last_clean":"2023-03-25T04:41:58.377821+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:41:58.377821+0000","last_undegraded":"2023-03-25T04:41:58.377821+0000","last_fullsized":"2023-03-25T04:41:58.377821+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:58.377836+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:41:58.377836+0000","last_peered":"2023-03-25T04:41:58.377836+0000","last_clean":"2023-03-25T04:41:58.377836+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:41:58.377836+0000","last_undegraded":"2023-03-25T04:41:58.377836+0000","last_fullsized":"2023-03-25T04:41:58.377836+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117206,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:44:01 2023","interfaces":[{"interface":"back","average":{"1min":0.27200000000000002,"5min":0.27000000000000002,"15min":0.26400000000000001},"min":{"1min":0.20399999999999999,"5min":0.17499999999999999,"15min":0.16},"max":{"1min":0.38400000000000001,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.216},{"interface":"front","average":{"1min":0.27600000000000002,"5min":0.27800000000000002,"15min":0.27100000000000002},"min":{"1min":0.23100000000000001,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.40400000000000003,"5min":0.47199999999999998,"15min":0.47199999999999998},"last":0.23999999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051796,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:43:55 2023","interfaces":[{"interface":"back","average":{"1min":0.26100000000000001,"5min":0.26700000000000002,"15min":0.27000000000000002},"min":{"1min":0.17799999999999999,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.374,"5min":0.44,"15min":0.44},"last":0.25600000000000001},{"interface":"front","average":{"1min":0.28299999999999997,"5min":0.28100000000000003,"15min":0.28399999999999997},"min":{"1min":0.20899999999999999,"5min":0.17599999999999999,"15min":0.16600000000000001},"max":{"1min":0.40100000000000002,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.28000000000000003}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:45:05.762 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:45:06.073 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all

Ignoring malparsed backtrace: 2023-03-25T04:45:25.658 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":624,"stamp":"2023-03-25T04:45:24.304294+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002366"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":41,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:42:34.382196+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:42:34.382196+0000","last_peered":"2023-03-25T04:42:34.382196+0000","last_clean":"2023-03-25T04:42:34.382196+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:42:34.382196+0000","last_undegraded":"2023-03-25T04:42:34.382196+0000","last_fullsized":"2023-03-25T04:42:34.382196+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:42:29.381744+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:42:29.381744+0000","last_peered":"2023-03-25T04:42:29.381744+0000","last_clean":"2023-03-25T04:42:29.381744+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:42:29.381744+0000","last_undegraded":"2023-03-25T04:42:29.381744+0000","last_fullsized":"2023-03-25T04:42:29.381744+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:23.375106+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:41:23.375106+0000","last_peered":"2023-03-25T04:41:23.375106+0000","last_clean":"2023-03-25T04:41:23.375106+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:41:23.375106+0000","last_undegraded":"2023-03-25T04:41:23.375106+0000","last_fullsized":"2023-03-25T04:41:23.375106+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:50.618930+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:41:50.618930+0000","last_peered":"2023-03-25T04:41:50.618930+0000","last_clean":"2023-03-25T04:41:50.618930+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:41:50.618930+0000","last_undegraded":"2023-03-25T04:41:50.618930+0000","last_fullsized":"2023-03-25T04:41:50.618930+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":41,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:58.377821+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:41:58.377821+0000","last_peered":"2023-03-25T04:41:58.377821+0000","last_clean":"2023-03-25T04:41:58.377821+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:41:58.377821+0000","last_undegraded":"2023-03-25T04:41:58.377821+0000","last_fullsized":"2023-03-25T04:41:58.377821+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:58.377836+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:41:58.377836+0000","last_peered":"2023-03-25T04:41:58.377836+0000","last_clean":"2023-03-25T04:41:58.377836+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:41:58.377836+0000","last_undegraded":"2023-03-25T04:41:58.377836+0000","last_fullsized":"2023-03-25T04:41:58.377836+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117213,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:45:01 2023","interfaces":[{"interface":"back","average":{"1min":0.251,"5min":0.26800000000000002,"15min":0.26400000000000001},"min":{"1min":0.183,"5min":0.17499999999999999,"15min":0.16},"max":{"1min":0.41499999999999998,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.26900000000000002},{"interface":"front","average":{"1min":0.28000000000000003,"5min":0.27900000000000003,"15min":0.27100000000000002},"min":{"1min":0.20999999999999999,"5min":0.185,"15min":0.17499999999999999},"max":{"1min":0.44400000000000001,"5min":0.47199999999999998,"15min":0.47199999999999998},"last":0.27900000000000003}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051803,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:44:55 2023","interfaces":[{"interface":"back","average":{"1min":0.252,"5min":0.26500000000000001,"15min":0.26900000000000002},"min":{"1min":0.183,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.39700000000000002,"5min":0.44,"15min":0.44},"last":0.20699999999999999},{"interface":"front","average":{"1min":0.27000000000000002,"5min":0.28100000000000003,"15min":0.28299999999999997},"min":{"1min":0.20300000000000001,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":0.42099999999999999,"5min":0.47599999999999998,"15min":0.48599999999999999},"last":0.23599999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:45:25.659 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:45:25.969 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:45:42.735+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:45:42.744+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:45:42.897 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace: 2023-03-25T04:46:00.183 INFO:teuthology.orchestra.run.smithi070.stdout:
, 2023-03-25T04:46:00.183 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":1217,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":20,"modified":"2023-03-25T04:45:42.307700+0000","services":{"mgr":{"daemons":{"summary":"","x":{"start_epoch":0,"start_stamp":"0.000000","gid":0,"addr":"(unrecognized address family 0)/0","metadata":{},"task_status":{}}}}}},"progress_events":{}}
, 2023-03-25T04:46:00.197 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now

Ignoring malparsed backtrace: 2023-03-25T04:46:19.910+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:46:20.039 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all

Ignoring malparsed backtrace: 2023-03-25T04:46:39.611 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}
, 2023-03-25T04:46:39.611 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json
, 2023-03-25T04:46:39.613+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:46:58.485+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:46:58.764+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:46:59.519+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:47:14.132 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:47:14.146 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":678,"stamp":"2023-03-25T04:47:12.325044+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002571"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":42,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:44.393426+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:44:44.393426+0000","last_peered":"2023-03-25T04:44:44.393426+0000","last_clean":"2023-03-25T04:44:44.393426+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:44:44.393426+0000","last_undegraded":"2023-03-25T04:44:44.393426+0000","last_fullsized":"2023-03-25T04:44:44.393426+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:39.393044+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:44:39.393044+0000","last_peered":"2023-03-25T04:44:39.393044+0000","last_clean":"2023-03-25T04:44:39.393044+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:44:39.393044+0000","last_undegraded":"2023-03-25T04:44:39.393044+0000","last_fullsized":"2023-03-25T04:44:39.393044+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:43:24.386177+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:43:24.386177+0000","last_peered":"2023-03-25T04:43:24.386177+0000","last_clean":"2023-03-25T04:43:24.386177+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:43:24.386177+0000","last_undegraded":"2023-03-25T04:43:24.386177+0000","last_fullsized":"2023-03-25T04:43:24.386177+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:43:55.629798+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:43:55.629798+0000","last_peered":"2023-03-25T04:43:55.629798+0000","last_clean":"2023-03-25T04:43:55.629798+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:43:55.629798+0000","last_undegraded":"2023-03-25T04:43:55.629798+0000","last_fullsized":"2023-03-25T04:43:55.629798+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":42,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:04.389904+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:44:04.389904+0000","last_peered":"2023-03-25T04:44:04.389904+0000","last_clean":"2023-03-25T04:44:04.389904+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:44:04.389904+0000","last_undegraded":"2023-03-25T04:44:04.389904+0000","last_fullsized":"2023-03-25T04:44:04.389904+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:04.389923+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:44:04.389923+0000","last_peered":"2023-03-25T04:44:04.389923+0000","last_clean":"2023-03-25T04:44:04.389923+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:44:04.389923+0000","last_undegraded":"2023-03-25T04:44:04.389923+0000","last_fullsized":"2023-03-25T04:44:04.389923+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117235,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:47:05 2023","interfaces":[{"interface":"back","average":{"1min":0.27800000000000002,"5min":0.27000000000000002,"15min":0.26300000000000001},"min":{"1min":0.20100000000000001,"5min":0.16900000000000001,"15min":0.16},"max":{"1min":0.39200000000000002,"5min":0.42999999999999999,"15min":0.45600000000000002},"last":0.379},{"interface":"front","average":{"1min":0.28399999999999997,"5min":0.28100000000000003,"15min":0.27200000000000002},"min":{"1min":0.17799999999999999,"5min":0.17799999999999999,"15min":0.17699999999999999},"max":{"1min":0.42599999999999999,"5min":0.47199999999999998,"15min":0.47199999999999998},"last":0.39800000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051825,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:47:01 2023","interfaces":[{"interface":"back","average":{"1min":0.27300000000000002,"5min":0.26100000000000001,"15min":0.27000000000000002},"min":{"1min":0.186,"5min":0.17399999999999999,"15min":0.17000000000000001},"max":{"1min":0.40400000000000003,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.23899999999999999},{"interface":"front","average":{"1min":0.28699999999999998,"5min":0.27500000000000002,"15min":0.28299999999999997},"min":{"1min":0.186,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.48599999999999999},"last":0.19600000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:47:31.198 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json
, 2023-03-25T04:47:31.411+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:47:31.789 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:47:49.509 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":696,"stamp":"2023-03-25T04:47:48.332093+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002697"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":42,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:44.393426+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:44:44.393426+0000","last_peered":"2023-03-25T04:44:44.393426+0000","last_clean":"2023-03-25T04:44:44.393426+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:44:44.393426+0000","last_undegraded":"2023-03-25T04:44:44.393426+0000","last_fullsized":"2023-03-25T04:44:44.393426+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:39.393044+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:44:39.393044+0000","last_peered":"2023-03-25T04:44:39.393044+0000","last_clean":"2023-03-25T04:44:39.393044+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:44:39.393044+0000","last_undegraded":"2023-03-25T04:44:39.393044+0000","last_fullsized":"2023-03-25T04:44:39.393044+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:45:24.396717+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:45:24.396717+0000","last_peered":"2023-03-25T04:45:24.396717+0000","last_clean":"2023-03-25T04:45:24.396717+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:45:24.396717+0000","last_undegraded":"2023-03-25T04:45:24.396717+0000","last_fullsized":"2023-03-25T04:45:24.396717+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:43:55.629798+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:43:55.629798+0000","last_peered":"2023-03-25T04:43:55.629798+0000","last_clean":"2023-03-25T04:43:55.629798+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:43:55.629798+0000","last_undegraded":"2023-03-25T04:43:55.629798+0000","last_fullsized":"2023-03-25T04:43:55.629798+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":42,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:04.389904+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:44:04.389904+0000","last_peered":"2023-03-25T04:44:04.389904+0000","last_clean":"2023-03-25T04:44:04.389904+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:44:04.389904+0000","last_undegraded":"2023-03-25T04:44:04.389904+0000","last_fullsized":"2023-03-25T04:44:04.389904+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:04.389923+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:44:04.389923+0000","last_peered":"2023-03-25T04:44:04.389923+0000","last_clean":"2023-03-25T04:44:04.389923+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:44:04.389923+0000","last_undegraded":"2023-03-25T04:44:04.389923+0000","last_fullsized":"2023-03-25T04:44:04.389923+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117242,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:47:05 2023","interfaces":[{"interface":"back","average":{"1min":0.27800000000000002,"5min":0.27000000000000002,"15min":0.26300000000000001},"min":{"1min":0.20100000000000001,"5min":0.16900000000000001,"15min":0.16},"max":{"1min":0.39200000000000002,"5min":0.42999999999999999,"15min":0.45600000000000002},"last":0.40500000000000003},{"interface":"front","average":{"1min":0.28399999999999997,"5min":0.28100000000000003,"15min":0.27200000000000002},"min":{"1min":0.17799999999999999,"5min":0.17799999999999999,"15min":0.17699999999999999},"max":{"1min":0.42599999999999999,"5min":0.47199999999999998,"15min":0.47199999999999998},"last":0.42399999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051834,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:47:01 2023","interfaces":[{"interface":"back","average":{"1min":0.27300000000000002,"5min":0.26100000000000001,"15min":0.27000000000000002},"min":{"1min":0.186,"5min":0.17399999999999999,"15min":0.17000000000000001},"max":{"1min":0.40400000000000003,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.25600000000000001},{"interface":"front","average":{"1min":0.28699999999999998,"5min":0.27500000000000002,"15min":0.28299999999999997},"min":{"1min":0.186,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.48599999999999999},"last":0.22800000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:47:49.510 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace:     ctx.managers[config['cluster']].wait_for_clean()
,   File "/home/teuthworker/src/github.com_ceph_ceph-c_511015c23eef0684a64171b710e432448df2b7c7/qa/tasks/ceph_manager.py", line 2747, in wait_for_clean
,     assert time.time() - start < timeout, \

Ignoring malparsed backtrace:  ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
,  1: /lib64/libpthread.so.0(+0x12cf0) [0x7f1453c2fcf0]
,  2: __pthread_mutex_lock()

Ignoring malparsed backtrace: 2023-03-25T04:28:05.172 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":142,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":5,"modified":"2023-03-25T04:27:12.092098+0000","services":{}},"progress_events":{}}
, 2023-03-25T04:28:05.185 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now
, 2023-03-25T04:28:05.186 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace: 2023-03-25T04:28:28.360 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":116,"stamp":"2023-03-25T04:28:28.108007+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002578"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.314661+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:27:45.314661+0000","last_peered":"2023-03-25T04:27:45.314661+0000","last_clean":"2023-03-25T04:27:45.314661+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:27:45.314661+0000","last_undegraded":"2023-03-25T04:27:45.314661+0000","last_fullsized":"2023-03-25T04:27:45.314661+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":25,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.310745+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:27:44.310745+0000","last_peered":"2023-03-25T04:27:44.310745+0000","last_clean":"2023-03-25T04:27:44.310745+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:27:44.310745+0000","last_undegraded":"2023-03-25T04:27:44.310745+0000","last_fullsized":"2023-03-25T04:27:44.310745+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":25,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.310650+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:27:44.310650+0000","last_peered":"2023-03-25T04:27:44.310650+0000","last_clean":"2023-03-25T04:27:44.310650+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:27:44.310650+0000","last_undegraded":"2023-03-25T04:27:44.310650+0000","last_fullsized":"2023-03-25T04:27:44.310650+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":26,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.312962+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:27:44.312962+0000","last_peered":"2023-03-25T04:27:44.312962+0000","last_clean":"2023-03-25T04:27:44.312962+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:27:44.312962+0000","last_undegraded":"2023-03-25T04:27:44.312962+0000","last_fullsized":"2023-03-25T04:27:44.312962+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315064+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:27:45.315064+0000","last_peered":"2023-03-25T04:27:45.315064+0000","last_clean":"2023-03-25T04:27:45.315064+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:27:45.315064+0000","last_undegraded":"2023-03-25T04:27:45.315064+0000","last_fullsized":"2023-03-25T04:27:45.315064+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":25,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.310707+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:27:44.310707+0000","last_peered":"2023-03-25T04:27:44.310707+0000","last_clean":"2023-03-25T04:27:44.310707+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:27:44.310707+0000","last_undegraded":"2023-03-25T04:27:44.310707+0000","last_fullsized":"2023-03-25T04:27:44.310707+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117007,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.375}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051597,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.186}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:28:28.362 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean
, 2023-03-25T04:28:28.362 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:28:45.655 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":124,"stamp":"2023-03-25T04:28:44.111265+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002492"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.314661+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:27:45.314661+0000","last_peered":"2023-03-25T04:27:45.314661+0000","last_clean":"2023-03-25T04:27:45.314661+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:27:45.314661+0000","last_undegraded":"2023-03-25T04:27:45.314661+0000","last_fullsized":"2023-03-25T04:27:45.314661+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":25,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.310745+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:27:44.310745+0000","last_peered":"2023-03-25T04:27:44.310745+0000","last_clean":"2023-03-25T04:27:44.310745+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:27:44.310745+0000","last_undegraded":"2023-03-25T04:27:44.310745+0000","last_fullsized":"2023-03-25T04:27:44.310745+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":25,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.310650+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:27:44.310650+0000","last_peered":"2023-03-25T04:27:44.310650+0000","last_clean":"2023-03-25T04:27:44.310650+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:27:44.310650+0000","last_undegraded":"2023-03-25T04:27:44.310650+0000","last_fullsized":"2023-03-25T04:27:44.310650+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":26,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.312962+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:27:44.312962+0000","last_peered":"2023-03-25T04:27:44.312962+0000","last_clean":"2023-03-25T04:27:44.312962+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:27:44.312962+0000","last_undegraded":"2023-03-25T04:27:44.312962+0000","last_fullsized":"2023-03-25T04:27:44.312962+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315064+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:27:45.315064+0000","last_peered":"2023-03-25T04:27:45.315064+0000","last_clean":"2023-03-25T04:27:45.315064+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:27:45.315064+0000","last_undegraded":"2023-03-25T04:27:45.315064+0000","last_fullsized":"2023-03-25T04:27:45.315064+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":25,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.310707+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:27:44.310707+0000","last_peered":"2023-03-25T04:27:44.310707+0000","last_clean":"2023-03-25T04:27:44.310707+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:27:44.310707+0000","last_undegraded":"2023-03-25T04:27:44.310707+0000","last_fullsized":"2023-03-25T04:27:44.310707+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117011,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:28:31 2023","interfaces":[{"interface":"back","average":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.36899999999999999},{"interface":"front","average":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"min":{"1min":0.19,"5min":0.19,"15min":0.19},"max":{"1min":0.46300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.39800000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051601,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:28:24 2023","interfaces":[{"interface":"back","average":{"1min":0.249,"5min":0.249,"15min":0.249},"min":{"1min":0.17699999999999999,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"last":0.254},{"interface":"front","average":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"min":{"1min":0.20200000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"last":0.312}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:28:45.656 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace: 2023-03-25T04:29:05.171 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}
, 2023-03-25T04:29:05.171 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json
, 2023-03-25T04:29:05.561 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:29:22.465 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:29:22.773 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all

Ignoring malparsed backtrace: 2023-03-25T04:29:42.405 INFO:teuthology.orchestra.run.smithi070.stdout:
, 2023-03-25T04:29:42.406 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":240,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":9,"modified":"2023-03-25T04:29:32.120358+0000","services":{"mon":{"daemons":{"summary":"","a":{"start_epoch":0,"start_stamp":"0.000000","gid":0,"addr":"(unrecognized address family 0)/0","metadata":{},"task_status":{}}}}}},"progress_events":{}}
, 2023-03-25T04:29:42.419 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:29:59.627 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":161,"stamp":"2023-03-25T04:29:58.125186+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002398"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315468+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:27:45.315468+0000","last_peered":"2023-03-25T04:27:45.315468+0000","last_clean":"2023-03-25T04:27:45.315468+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:27:45.315468+0000","last_undegraded":"2023-03-25T04:27:45.315468+0000","last_fullsized":"2023-03-25T04:27:45.315468+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312845+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:27:45.312845+0000","last_peered":"2023-03-25T04:27:45.312845+0000","last_clean":"2023-03-25T04:27:45.312845+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:27:45.312845+0000","last_undegraded":"2023-03-25T04:27:45.312845+0000","last_fullsized":"2023-03-25T04:27:45.312845+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312882+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:27:45.312882+0000","last_peered":"2023-03-25T04:27:45.312882+0000","last_clean":"2023-03-25T04:27:45.312882+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:27:45.312882+0000","last_undegraded":"2023-03-25T04:27:45.312882+0000","last_fullsized":"2023-03-25T04:27:45.312882+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.311730+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:27:45.311730+0000","last_peered":"2023-03-25T04:27:45.311730+0000","last_clean":"2023-03-25T04:27:45.311730+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:27:45.311730+0000","last_undegraded":"2023-03-25T04:27:45.311730+0000","last_fullsized":"2023-03-25T04:27:45.311730+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315735+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:27:45.315735+0000","last_peered":"2023-03-25T04:27:45.315735+0000","last_clean":"2023-03-25T04:27:45.315735+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:27:45.315735+0000","last_undegraded":"2023-03-25T04:27:45.315735+0000","last_fullsized":"2023-03-25T04:27:45.315735+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312833+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:27:45.312833+0000","last_peered":"2023-03-25T04:27:45.312833+0000","last_clean":"2023-03-25T04:27:45.312833+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:27:45.312833+0000","last_undegraded":"2023-03-25T04:27:45.312833+0000","last_fullsized":"2023-03-25T04:27:45.312833+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117026,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:29:33 2023","interfaces":[{"interface":"back","average":{"1min":0.26300000000000001,"5min":0.29099999999999998,"15min":0.29599999999999999},"min":{"1min":0.191,"5min":0.191,"15min":0.191},"max":{"1min":0.371,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.20499999999999999},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.29599999999999999,"15min":0.29999999999999999},"min":{"1min":0.21099999999999999,"5min":0.19,"15min":0.19},"max":{"1min":0.39800000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.19500000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051616,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:29:29 2023","interfaces":[{"interface":"back","average":{"1min":0.27200000000000002,"5min":0.253,"15min":0.25},"min":{"1min":0.183,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"last":0.26400000000000001},{"interface":"front","average":{"1min":0.28499999999999998,"5min":0.26800000000000002,"15min":0.26500000000000001},"min":{"1min":0.16600000000000001,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"last":0.29899999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:29:59.629 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean

Ignoring malparsed backtrace: 2023-03-25T04:30:16.889 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:30:16.903 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":170,"stamp":"2023-03-25T04:30:16.128427+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002222"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315468+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:27:45.315468+0000","last_peered":"2023-03-25T04:27:45.315468+0000","last_clean":"2023-03-25T04:27:45.315468+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:27:45.315468+0000","last_undegraded":"2023-03-25T04:27:45.315468+0000","last_fullsized":"2023-03-25T04:27:45.315468+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312845+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:27:45.312845+0000","last_peered":"2023-03-25T04:27:45.312845+0000","last_clean":"2023-03-25T04:27:45.312845+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:27:45.312845+0000","last_undegraded":"2023-03-25T04:27:45.312845+0000","last_fullsized":"2023-03-25T04:27:45.312845+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312882+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:27:45.312882+0000","last_peered":"2023-03-25T04:27:45.312882+0000","last_clean":"2023-03-25T04:27:45.312882+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:27:45.312882+0000","last_undegraded":"2023-03-25T04:27:45.312882+0000","last_fullsized":"2023-03-25T04:27:45.312882+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.311730+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:27:45.311730+0000","last_peered":"2023-03-25T04:27:45.311730+0000","last_clean":"2023-03-25T04:27:45.311730+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:27:45.311730+0000","last_undegraded":"2023-03-25T04:27:45.311730+0000","last_fullsized":"2023-03-25T04:27:45.311730+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315735+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:27:45.315735+0000","last_peered":"2023-03-25T04:27:45.315735+0000","last_clean":"2023-03-25T04:27:45.315735+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:27:45.315735+0000","last_undegraded":"2023-03-25T04:27:45.315735+0000","last_fullsized":"2023-03-25T04:27:45.315735+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312833+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:27:45.312833+0000","last_peered":"2023-03-25T04:27:45.312833+0000","last_clean":"2023-03-25T04:27:45.312833+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:27:45.312833+0000","last_undegraded":"2023-03-25T04:27:45.312833+0000","last_fullsized":"2023-03-25T04:27:45.312833+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117030,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:29:33 2023","interfaces":[{"interface":"back","average":{"1min":0.26300000000000001,"5min":0.29099999999999998,"15min":0.29599999999999999},"min":{"1min":0.191,"5min":0.191,"15min":0.191},"max":{"1min":0.371,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.245},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.29599999999999999,"15min":0.29999999999999999},"min":{"1min":0.21099999999999999,"5min":0.19,"15min":0.19},"max":{"1min":0.39800000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.20899999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051620,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:29:29 2023","interfaces":[{"interface":"back","average":{"1min":0.27200000000000002,"5min":0.253,"15min":0.25},"min":{"1min":0.183,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"last":0.19400000000000001},{"interface":"front","average":{"1min":0.28499999999999998,"5min":0.26800000000000002,"15min":0.26500000000000001},"min":{"1min":0.16600000000000001,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"last":0.23599999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:30:31.984+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:30:32.201+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:30:32.981+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)

Ignoring malparsed backtrace: 2023-03-25T04:30:48.449 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now
, 2023-03-25T04:30:48.450 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:30:48.760 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:31:08.342 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":196,"stamp":"2023-03-25T04:31:08.139018+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002431"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315468+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:27:45.315468+0000","last_peered":"2023-03-25T04:27:45.315468+0000","last_clean":"2023-03-25T04:27:45.315468+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:27:45.315468+0000","last_undegraded":"2023-03-25T04:27:45.315468+0000","last_fullsized":"2023-03-25T04:27:45.315468+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312845+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:27:45.312845+0000","last_peered":"2023-03-25T04:27:45.312845+0000","last_clean":"2023-03-25T04:27:45.312845+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:27:45.312845+0000","last_undegraded":"2023-03-25T04:27:45.312845+0000","last_fullsized":"2023-03-25T04:27:45.312845+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312882+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:27:45.312882+0000","last_peered":"2023-03-25T04:27:45.312882+0000","last_clean":"2023-03-25T04:27:45.312882+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:27:45.312882+0000","last_undegraded":"2023-03-25T04:27:45.312882+0000","last_fullsized":"2023-03-25T04:27:45.312882+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.311730+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:27:45.311730+0000","last_peered":"2023-03-25T04:27:45.311730+0000","last_clean":"2023-03-25T04:27:45.311730+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:27:45.311730+0000","last_undegraded":"2023-03-25T04:27:45.311730+0000","last_fullsized":"2023-03-25T04:27:45.311730+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315735+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:27:45.315735+0000","last_peered":"2023-03-25T04:27:45.315735+0000","last_clean":"2023-03-25T04:27:45.315735+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:27:45.315735+0000","last_undegraded":"2023-03-25T04:27:45.315735+0000","last_fullsized":"2023-03-25T04:27:45.315735+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312833+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:27:45.312833+0000","last_peered":"2023-03-25T04:27:45.312833+0000","last_clean":"2023-03-25T04:27:45.312833+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:27:45.312833+0000","last_undegraded":"2023-03-25T04:27:45.312833+0000","last_fullsized":"2023-03-25T04:27:45.312833+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117040,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:30:37 2023","interfaces":[{"interface":"back","average":{"1min":0.28799999999999998,"5min":0.28899999999999998,"15min":0.29499999999999998},"min":{"1min":0.20200000000000001,"5min":0.191,"15min":0.191},"max":{"1min":0.433,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.29799999999999999},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.28999999999999998,"15min":0.29799999999999999},"min":{"1min":0.17499999999999999,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":0.40300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.25700000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051630,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:30:32 2023","interfaces":[{"interface":"back","average":{"1min":0.25600000000000001,"5min":0.255,"15min":0.251},"min":{"1min":0.19400000000000001,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.35699999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"last":0.22800000000000001},{"interface":"front","average":{"1min":0.28699999999999998,"5min":0.27200000000000002,"15min":0.26600000000000001},"min":{"1min":0.20599999999999999,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.39400000000000002,"5min":0.42899999999999999,"15min":0.42899999999999999},"last":0.24099999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:31:08.343 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace: 2023-03-25T04:31:24.283+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:31:25.192+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:31:25.295+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)

Ignoring malparsed backtrace: 2023-03-25T04:31:40.257 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:31:40.271 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":212,"stamp":"2023-03-25T04:31:40.144732+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002208"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.315468+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:27:45.315468+0000","last_peered":"2023-03-25T04:27:45.315468+0000","last_clean":"2023-03-25T04:27:45.315468+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:27:45.315468+0000","last_undegraded":"2023-03-25T04:27:45.315468+0000","last_fullsized":"2023-03-25T04:27:45.315468+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":26,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:27:45.312845+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:27:45.312845+0000","last_peered":"2023-03-25T04:27:45.312845+0000","last_clean":"2023-03-25T04:27:45.312845+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:27:45.312845+0000","last_undegraded":"2023-03-25T04:27:45.312845+0000","last_fullsized":"2023-03-25T04:27:45.312845+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:23.308607+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:29:23.308607+0000","last_peered":"2023-03-25T04:29:23.308607+0000","last_clean":"2023-03-25T04:29:23.308607+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:29:23.308607+0000","last_undegraded":"2023-03-25T04:29:23.308607+0000","last_fullsized":"2023-03-25T04:29:23.308607+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":28,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:20.554710+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:29:20.554710+0000","last_peered":"2023-03-25T04:29:20.554710+0000","last_clean":"2023-03-25T04:29:20.554710+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:29:20.554710+0000","last_undegraded":"2023-03-25T04:29:20.554710+0000","last_fullsized":"2023-03-25T04:29:20.554710+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309216+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:29:28.309216+0000","last_peered":"2023-03-25T04:29:28.309216+0000","last_clean":"2023-03-25T04:29:28.309216+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:29:28.309216+0000","last_undegraded":"2023-03-25T04:29:28.309216+0000","last_fullsized":"2023-03-25T04:29:28.309216+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309233+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:29:28.309233+0000","last_peered":"2023-03-25T04:29:28.309233+0000","last_clean":"2023-03-25T04:29:28.309233+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:29:28.309233+0000","last_undegraded":"2023-03-25T04:29:28.309233+0000","last_fullsized":"2023-03-25T04:29:28.309233+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117046,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:30:37 2023","interfaces":[{"interface":"back","average":{"1min":0.28799999999999998,"5min":0.28899999999999998,"15min":0.29499999999999998},"min":{"1min":0.20200000000000001,"5min":0.191,"15min":0.191},"max":{"1min":0.433,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.216},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.28999999999999998,"15min":0.29799999999999999},"min":{"1min":0.17499999999999999,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":0.40300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.24299999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051637,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:31:35 2023","interfaces":[{"interface":"back","average":{"1min":0.26100000000000001,"5min":0.25700000000000001,"15min":0.251},"min":{"1min":0.19,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.35899999999999999,"5min":0.41599999999999998,"15min":0.41599999999999998},"last":0.221},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.27400000000000002,"15min":0.26700000000000002},"min":{"1min":0.20200000000000001,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.374,"5min":0.42899999999999999,"15min":0.42899999999999999},"last":0.24099999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:32:01.906+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:32:02.388+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:32:02.794 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:32:22.326 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":233,"stamp":"2023-03-25T04:32:22.153289+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002545"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:33.309748+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:29:33.309748+0000","last_peered":"2023-03-25T04:29:33.309748+0000","last_clean":"2023-03-25T04:29:33.309748+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:29:33.309748+0000","last_undegraded":"2023-03-25T04:29:33.309748+0000","last_fullsized":"2023-03-25T04:29:33.309748+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:33.309729+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:29:33.309729+0000","last_peered":"2023-03-25T04:29:33.309729+0000","last_clean":"2023-03-25T04:29:33.309729+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:29:33.309729+0000","last_undegraded":"2023-03-25T04:29:33.309729+0000","last_fullsized":"2023-03-25T04:29:33.309729+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:23.308607+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:29:23.308607+0000","last_peered":"2023-03-25T04:29:23.308607+0000","last_clean":"2023-03-25T04:29:23.308607+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:29:23.308607+0000","last_undegraded":"2023-03-25T04:29:23.308607+0000","last_fullsized":"2023-03-25T04:29:23.308607+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":28,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:20.554710+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:29:20.554710+0000","last_peered":"2023-03-25T04:29:20.554710+0000","last_clean":"2023-03-25T04:29:20.554710+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:29:20.554710+0000","last_undegraded":"2023-03-25T04:29:20.554710+0000","last_fullsized":"2023-03-25T04:29:20.554710+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309216+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:29:28.309216+0000","last_peered":"2023-03-25T04:29:28.309216+0000","last_clean":"2023-03-25T04:29:28.309216+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:29:28.309216+0000","last_undegraded":"2023-03-25T04:29:28.309216+0000","last_fullsized":"2023-03-25T04:29:28.309216+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309233+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:29:28.309233+0000","last_peered":"2023-03-25T04:29:28.309233+0000","last_clean":"2023-03-25T04:29:28.309233+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:29:28.309233+0000","last_undegraded":"2023-03-25T04:29:28.309233+0000","last_fullsized":"2023-03-25T04:29:28.309233+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117055,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:31:42 2023","interfaces":[{"interface":"back","average":{"1min":0.27500000000000002,"5min":0.28399999999999997,"15min":0.29399999999999998},"min":{"1min":0.19800000000000001,"5min":0.191,"15min":0.191},"max":{"1min":0.36899999999999999,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.32300000000000001},{"interface":"front","average":{"1min":0.27900000000000003,"5min":0.28499999999999998,"15min":0.29599999999999999},"min":{"1min":0.216,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":0.40799999999999997,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.27100000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051646,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:31:35 2023","interfaces":[{"interface":"back","average":{"1min":0.26100000000000001,"5min":0.25700000000000001,"15min":0.251},"min":{"1min":0.19,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.35899999999999999,"5min":0.41599999999999998,"15min":0.41599999999999998},"last":0.25600000000000001},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.27400000000000002,"15min":0.26700000000000002},"min":{"1min":0.20200000000000001,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.374,"5min":0.42899999999999999,"15min":0.42899999999999999},"last":0.29199999999999998}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:32:22.327 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean
, 2023-03-25T04:32:22.327 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}

Ignoring malparsed backtrace: 2023-03-25T04:32:39.836 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:32:39.849 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":241,"stamp":"2023-03-25T04:32:38.156312+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002346"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:33.309748+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:29:33.309748+0000","last_peered":"2023-03-25T04:29:33.309748+0000","last_clean":"2023-03-25T04:29:33.309748+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:29:33.309748+0000","last_undegraded":"2023-03-25T04:29:33.309748+0000","last_fullsized":"2023-03-25T04:29:33.309748+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:33.309729+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:29:33.309729+0000","last_peered":"2023-03-25T04:29:33.309729+0000","last_clean":"2023-03-25T04:29:33.309729+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:29:33.309729+0000","last_undegraded":"2023-03-25T04:29:33.309729+0000","last_fullsized":"2023-03-25T04:29:33.309729+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:23.308607+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:29:23.308607+0000","last_peered":"2023-03-25T04:29:23.308607+0000","last_clean":"2023-03-25T04:29:23.308607+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:29:23.308607+0000","last_undegraded":"2023-03-25T04:29:23.308607+0000","last_fullsized":"2023-03-25T04:29:23.308607+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":28,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:20.554710+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:29:20.554710+0000","last_peered":"2023-03-25T04:29:20.554710+0000","last_clean":"2023-03-25T04:29:20.554710+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:29:20.554710+0000","last_undegraded":"2023-03-25T04:29:20.554710+0000","last_fullsized":"2023-03-25T04:29:20.554710+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309216+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:29:28.309216+0000","last_peered":"2023-03-25T04:29:28.309216+0000","last_clean":"2023-03-25T04:29:28.309216+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:29:28.309216+0000","last_undegraded":"2023-03-25T04:29:28.309216+0000","last_fullsized":"2023-03-25T04:29:28.309216+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309233+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:29:28.309233+0000","last_peered":"2023-03-25T04:29:28.309233+0000","last_clean":"2023-03-25T04:29:28.309233+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:29:28.309233+0000","last_undegraded":"2023-03-25T04:29:28.309233+0000","last_fullsized":"2023-03-25T04:29:28.309233+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117058,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:31:42 2023","interfaces":[{"interface":"back","average":{"1min":0.27500000000000002,"5min":0.28399999999999997,"15min":0.29399999999999998},"min":{"1min":0.19800000000000001,"5min":0.191,"15min":0.191},"max":{"1min":0.36899999999999999,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.30599999999999999},{"interface":"front","average":{"1min":0.27900000000000003,"5min":0.28499999999999998,"15min":0.29599999999999999},"min":{"1min":0.216,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":0.40799999999999997,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.32500000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051649,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:31:35 2023","interfaces":[{"interface":"back","average":{"1min":0.26100000000000001,"5min":0.25700000000000001,"15min":0.251},"min":{"1min":0.19,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.35899999999999999,"5min":0.41599999999999998,"15min":0.41599999999999998},"last":0.24199999999999999},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.27400000000000002,"15min":0.26700000000000002},"min":{"1min":0.20200000000000001,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.374,"5min":0.42899999999999999,"15min":0.42899999999999999},"last":0.22600000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:32:54.944+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:32:55.469+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:32:55.927+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)

Ignoring malparsed backtrace: 2023-03-25T04:33:13.958 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean
, 2023-03-25T04:33:13.959 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}
, 2023-03-25T04:33:13.959 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json

Ignoring malparsed backtrace: 2023-03-25T04:33:31.199 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":267,"stamp":"2023-03-25T04:33:30.166660+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002269"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:33.309748+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:29:33.309748+0000","last_peered":"2023-03-25T04:29:33.309748+0000","last_clean":"2023-03-25T04:29:33.309748+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:29:33.309748+0000","last_undegraded":"2023-03-25T04:29:33.309748+0000","last_fullsized":"2023-03-25T04:29:33.309748+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:33.309729+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:29:33.309729+0000","last_peered":"2023-03-25T04:29:33.309729+0000","last_clean":"2023-03-25T04:29:33.309729+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:29:33.309729+0000","last_undegraded":"2023-03-25T04:29:33.309729+0000","last_fullsized":"2023-03-25T04:29:33.309729+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":28,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:31:23.320894+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:31:23.320894+0000","last_peered":"2023-03-25T04:31:23.320894+0000","last_clean":"2023-03-25T04:31:23.320894+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:31:23.320894+0000","last_undegraded":"2023-03-25T04:31:23.320894+0000","last_fullsized":"2023-03-25T04:31:23.320894+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":28,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:20.554710+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:29:20.554710+0000","last_peered":"2023-03-25T04:29:20.554710+0000","last_clean":"2023-03-25T04:29:20.554710+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:29:20.554710+0000","last_undegraded":"2023-03-25T04:29:20.554710+0000","last_fullsized":"2023-03-25T04:29:20.554710+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309216+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:29:28.309216+0000","last_peered":"2023-03-25T04:29:28.309216+0000","last_clean":"2023-03-25T04:29:28.309216+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:29:28.309216+0000","last_undegraded":"2023-03-25T04:29:28.309216+0000","last_fullsized":"2023-03-25T04:29:28.309216+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":27,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:29:28.309233+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:29:28.309233+0000","last_peered":"2023-03-25T04:29:28.309233+0000","last_clean":"2023-03-25T04:29:28.309233+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:29:28.309233+0000","last_undegraded":"2023-03-25T04:29:28.309233+0000","last_fullsized":"2023-03-25T04:29:28.309233+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117068,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:32:46 2023","interfaces":[{"interface":"back","average":{"1min":0.27000000000000002,"5min":0.27900000000000003,"15min":0.29199999999999998},"min":{"1min":0.17899999999999999,"5min":0.17899999999999999,"15min":0.17899999999999999},"max":{"1min":0.35599999999999998,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.28799999999999998},{"interface":"front","average":{"1min":0.26800000000000002,"5min":0.27800000000000002,"15min":0.29399999999999998},"min":{"1min":0.19900000000000001,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":0.377,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.315}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051660,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:32:36 2023","interfaces":[{"interface":"back","average":{"1min":0.248,"5min":0.25700000000000001,"15min":0.251},"min":{"1min":0.17000000000000001,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"last":0.22700000000000001},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.27600000000000002,"15min":0.26800000000000002},"min":{"1min":0.192,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"last":0.20399999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:33:31.201 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:33:31.400+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)

Ignoring malparsed backtrace: 2023-03-25T04:33:47.531+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:33:47.951+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:33:48.505+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)

Ignoring malparsed backtrace: 2023-03-25T04:34:06.104 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json
, 2023-03-25T04:34:06.118+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:34:06.497 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:34:23.399 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:34:23.665+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:34:23.712 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:34:40.195+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:34:40.523+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:34:40.676 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace: 2023-03-25T04:35:03.381+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:35:04.139+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:35:04.391+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)

Ignoring malparsed backtrace: 2023-03-25T04:35:22.738 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean
, 2023-03-25T04:35:22.739 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}
, 2023-03-25T04:35:22.739 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json

Ignoring malparsed backtrace: 2023-03-25T04:35:41.393+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:35:42.352+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:35:42.376+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:35:57.384 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":340,"stamp":"2023-03-25T04:35:56.194992+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.001930"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":36,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:31:43.322894+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:31:43.322894+0000","last_peered":"2023-03-25T04:31:43.322894+0000","last_clean":"2023-03-25T04:31:43.322894+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:31:43.322894+0000","last_undegraded":"2023-03-25T04:31:43.322894+0000","last_fullsized":"2023-03-25T04:31:43.322894+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":28,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:31:43.322876+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:31:43.322876+0000","last_peered":"2023-03-25T04:31:43.322876+0000","last_clean":"2023-03-25T04:31:43.322876+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:31:43.322876+0000","last_undegraded":"2023-03-25T04:31:43.322876+0000","last_fullsized":"2023-03-25T04:31:43.322876+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:23.331998+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:33:23.331998+0000","last_peered":"2023-03-25T04:33:23.331998+0000","last_clean":"2023-03-25T04:33:23.331998+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:33:23.331998+0000","last_undegraded":"2023-03-25T04:33:23.331998+0000","last_fullsized":"2023-03-25T04:33:23.331998+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:30.576723+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:33:30.576723+0000","last_peered":"2023-03-25T04:33:30.576723+0000","last_clean":"2023-03-25T04:33:30.576723+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:33:30.576723+0000","last_undegraded":"2023-03-25T04:33:30.576723+0000","last_fullsized":"2023-03-25T04:33:30.576723+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":37,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:38.333447+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:33:38.333447+0000","last_peered":"2023-03-25T04:33:38.333447+0000","last_clean":"2023-03-25T04:33:38.333447+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:33:38.333447+0000","last_undegraded":"2023-03-25T04:33:38.333447+0000","last_fullsized":"2023-03-25T04:33:38.333447+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:38.333467+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:33:38.333467+0000","last_peered":"2023-03-25T04:33:38.333467+0000","last_clean":"2023-03-25T04:33:38.333467+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:33:38.333467+0000","last_undegraded":"2023-03-25T04:33:38.333467+0000","last_fullsized":"2023-03-25T04:33:38.333467+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117098,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:35:49 2023","interfaces":[{"interface":"back","average":{"1min":0.255,"5min":0.26000000000000001,"15min":0.28199999999999997},"min":{"1min":0.17899999999999999,"5min":0.16,"15min":0.16},"max":{"1min":0.36399999999999999,"5min":0.42599999999999999,"15min":0.45100000000000001},"last":0.26100000000000001},{"interface":"front","average":{"1min":0.26000000000000001,"5min":0.26700000000000002,"15min":0.28599999999999998},"min":{"1min":0.188,"5min":0.17999999999999999,"15min":0.17499999999999999},"max":{"1min":0.39100000000000001,"5min":0.45200000000000001,"15min":0.46300000000000002},"last":0.28399999999999997}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051689,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:35:42 2023","interfaces":[{"interface":"back","average":{"1min":0.29399999999999998,"5min":0.27200000000000002,"15min":0.25800000000000001},"min":{"1min":0.17299999999999999,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.41099999999999998,"5min":0.42699999999999999,"15min":0.42699999999999999},"last":0.20399999999999999},{"interface":"front","average":{"1min":0.30299999999999999,"5min":0.28399999999999997,"15min":0.27300000000000002},"min":{"1min":0.19700000000000001,"5min":0.192,"15min":0.16600000000000001},"max":{"1min":0.43099999999999999,"5min":0.45800000000000002,"15min":0.45800000000000002},"last":0.22}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:35:57.624+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)

Ignoring malparsed backtrace: 2023-03-25T04:36:17.226 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:36:17.239 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":350,"stamp":"2023-03-25T04:36:16.198866+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002148"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":37,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:53.335026+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:33:53.335026+0000","last_peered":"2023-03-25T04:33:53.335026+0000","last_clean":"2023-03-25T04:33:53.335026+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:33:53.335026+0000","last_undegraded":"2023-03-25T04:33:53.335026+0000","last_fullsized":"2023-03-25T04:33:53.335026+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:53.335009+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:33:53.335009+0000","last_peered":"2023-03-25T04:33:53.335009+0000","last_clean":"2023-03-25T04:33:53.335009+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:33:53.335009+0000","last_undegraded":"2023-03-25T04:33:53.335009+0000","last_fullsized":"2023-03-25T04:33:53.335009+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:23.331998+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:33:23.331998+0000","last_peered":"2023-03-25T04:33:23.331998+0000","last_clean":"2023-03-25T04:33:23.331998+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:33:23.331998+0000","last_undegraded":"2023-03-25T04:33:23.331998+0000","last_fullsized":"2023-03-25T04:33:23.331998+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:30.576723+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:33:30.576723+0000","last_peered":"2023-03-25T04:33:30.576723+0000","last_clean":"2023-03-25T04:33:30.576723+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:33:30.576723+0000","last_undegraded":"2023-03-25T04:33:30.576723+0000","last_fullsized":"2023-03-25T04:33:30.576723+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":37,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:38.333447+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:33:38.333447+0000","last_peered":"2023-03-25T04:33:38.333447+0000","last_clean":"2023-03-25T04:33:38.333447+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:33:38.333447+0000","last_undegraded":"2023-03-25T04:33:38.333447+0000","last_fullsized":"2023-03-25T04:33:38.333447+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:38.333467+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:33:38.333467+0000","last_peered":"2023-03-25T04:33:38.333467+0000","last_clean":"2023-03-25T04:33:38.333467+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:33:38.333467+0000","last_undegraded":"2023-03-25T04:33:38.333467+0000","last_fullsized":"2023-03-25T04:33:38.333467+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117102,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:35:49 2023","interfaces":[{"interface":"back","average":{"1min":0.255,"5min":0.26000000000000001,"15min":0.28199999999999997},"min":{"1min":0.17899999999999999,"5min":0.16,"15min":0.16},"max":{"1min":0.36399999999999999,"5min":0.42599999999999999,"15min":0.45100000000000001},"last":0.28399999999999997},{"interface":"front","average":{"1min":0.26000000000000001,"5min":0.26700000000000002,"15min":0.28599999999999998},"min":{"1min":0.188,"5min":0.17999999999999999,"15min":0.17499999999999999},"max":{"1min":0.39100000000000001,"5min":0.45200000000000001,"15min":0.46300000000000002},"last":0.30299999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051693,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:35:42 2023","interfaces":[{"interface":"back","average":{"1min":0.29399999999999998,"5min":0.27200000000000002,"15min":0.25800000000000001},"min":{"1min":0.17299999999999999,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.41099999999999998,"5min":0.42699999999999999,"15min":0.42699999999999999},"last":0.35099999999999998},{"interface":"front","average":{"1min":0.30299999999999999,"5min":0.28399999999999997,"15min":0.27300000000000002},"min":{"1min":0.19700000000000001,"5min":0.192,"15min":0.16600000000000001},"max":{"1min":0.43099999999999999,"5min":0.45800000000000002,"15min":0.45800000000000002},"last":0.32900000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:36:34.556+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)
, 2023-03-25T04:36:34.717+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:36:35.583+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)

Ignoring malparsed backtrace: 2023-03-25T04:36:52.680 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json
, 2023-03-25T04:36:52.701+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:36:53.067 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:37:09.963 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:37:10.273 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all

Ignoring malparsed backtrace: 2023-03-25T04:37:26.856+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:37:27.288 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:37:27.538+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:37:41.903 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":392,"stamp":"2023-03-25T04:37:40.213373+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002012"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":37,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:53.335026+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:33:53.335026+0000","last_peered":"2023-03-25T04:33:53.335026+0000","last_clean":"2023-03-25T04:33:53.335026+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:33:53.335026+0000","last_undegraded":"2023-03-25T04:33:53.335026+0000","last_fullsized":"2023-03-25T04:33:53.335026+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:53.335009+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:33:53.335009+0000","last_peered":"2023-03-25T04:33:53.335009+0000","last_clean":"2023-03-25T04:33:53.335009+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:33:53.335009+0000","last_undegraded":"2023-03-25T04:33:53.335009+0000","last_fullsized":"2023-03-25T04:33:53.335009+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:23.343585+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:35:23.343585+0000","last_peered":"2023-03-25T04:35:23.343585+0000","last_clean":"2023-03-25T04:35:23.343585+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:35:23.343585+0000","last_undegraded":"2023-03-25T04:35:23.343585+0000","last_fullsized":"2023-03-25T04:35:23.343585+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:30.576723+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:33:30.576723+0000","last_peered":"2023-03-25T04:33:30.576723+0000","last_clean":"2023-03-25T04:33:30.576723+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:33:30.576723+0000","last_undegraded":"2023-03-25T04:33:30.576723+0000","last_fullsized":"2023-03-25T04:33:30.576723+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":37,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:38.333447+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:33:38.333447+0000","last_peered":"2023-03-25T04:33:38.333447+0000","last_clean":"2023-03-25T04:33:38.333447+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:33:38.333447+0000","last_undegraded":"2023-03-25T04:33:38.333447+0000","last_fullsized":"2023-03-25T04:33:38.333447+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":29,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:33:38.333467+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:33:38.333467+0000","last_peered":"2023-03-25T04:33:38.333467+0000","last_clean":"2023-03-25T04:33:38.333467+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:33:38.333467+0000","last_undegraded":"2023-03-25T04:33:38.333467+0000","last_fullsized":"2023-03-25T04:33:38.333467+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117119,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:36:50 2023","interfaces":[{"interface":"back","average":{"1min":0.254,"5min":0.25600000000000001,"15min":0.27900000000000003},"min":{"1min":0.20499999999999999,"5min":0.16,"15min":0.16},"max":{"1min":0.371,"5min":0.42599999999999999,"15min":0.45100000000000001},"last":0.23000000000000001},{"interface":"front","average":{"1min":0.26800000000000002,"5min":0.26500000000000001,"15min":0.28399999999999997},"min":{"1min":0.19700000000000001,"5min":0.17999999999999999,"15min":0.17499999999999999},"max":{"1min":0.34699999999999998,"5min":0.45200000000000001,"15min":0.46300000000000002},"last":0.20899999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051710,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:36:44 2023","interfaces":[{"interface":"back","average":{"1min":0.28399999999999997,"5min":0.27700000000000002,"15min":0.26100000000000001},"min":{"1min":0.192,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.40899999999999997,"5min":0.42699999999999999,"15min":0.42699999999999999},"last":0.39700000000000002},{"interface":"front","average":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.27500000000000002},"min":{"1min":0.182,"5min":0.182,"15min":0.16600000000000001},"max":{"1min":0.439,"5min":0.45800000000000002,"15min":0.45800000000000002},"last":0.439}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:37:42.400+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)

Ignoring malparsed backtrace: 2023-03-25T04:38:04.766+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:38:04.922 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:38:05.246 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:38:19.605 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":411,"stamp":"2023-03-25T04:38:18.220966+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002368"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":38,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:36:03.347070+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:36:03.347070+0000","last_peered":"2023-03-25T04:36:03.347070+0000","last_clean":"2023-03-25T04:36:03.347070+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:36:03.347070+0000","last_undegraded":"2023-03-25T04:36:03.347070+0000","last_fullsized":"2023-03-25T04:36:03.347070+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:36:03.347025+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:36:03.347025+0000","last_peered":"2023-03-25T04:36:03.347025+0000","last_clean":"2023-03-25T04:36:03.347025+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:36:03.347025+0000","last_undegraded":"2023-03-25T04:36:03.347025+0000","last_fullsized":"2023-03-25T04:36:03.347025+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:23.343585+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:35:23.343585+0000","last_peered":"2023-03-25T04:35:23.343585+0000","last_clean":"2023-03-25T04:35:23.343585+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:35:23.343585+0000","last_undegraded":"2023-03-25T04:35:23.343585+0000","last_fullsized":"2023-03-25T04:35:23.343585+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:35.587025+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:35:35.587025+0000","last_peered":"2023-03-25T04:35:35.587025+0000","last_clean":"2023-03-25T04:35:35.587025+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:35:35.587025+0000","last_undegraded":"2023-03-25T04:35:35.587025+0000","last_fullsized":"2023-03-25T04:35:35.587025+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":38,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:43.345308+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:35:43.345308+0000","last_peered":"2023-03-25T04:35:43.345308+0000","last_clean":"2023-03-25T04:35:43.345308+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:35:43.345308+0000","last_undegraded":"2023-03-25T04:35:43.345308+0000","last_fullsized":"2023-03-25T04:35:43.345308+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:43.345324+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:35:43.345324+0000","last_peered":"2023-03-25T04:35:43.345324+0000","last_clean":"2023-03-25T04:35:43.345324+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:35:43.345324+0000","last_undegraded":"2023-03-25T04:35:43.345324+0000","last_fullsized":"2023-03-25T04:35:43.345324+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117127,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:37:53 2023","interfaces":[{"interface":"back","average":{"1min":0.25800000000000001,"5min":0.253,"15min":0.27700000000000002},"min":{"1min":0.184,"5min":0.16,"15min":0.16},"max":{"1min":0.44600000000000001,"5min":0.44600000000000001,"15min":0.45100000000000001},"last":0.35199999999999998},{"interface":"front","average":{"1min":0.26900000000000002,"5min":0.26500000000000001,"15min":0.28199999999999997},"min":{"1min":0.192,"5min":0.17999999999999999,"15min":0.17499999999999999},"max":{"1min":0.46400000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"last":0.33200000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051717,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:37:44 2023","interfaces":[{"interface":"back","average":{"1min":0.27300000000000002,"5min":0.28199999999999997,"15min":0.26200000000000001},"min":{"1min":0.17899999999999999,"5min":0.17299999999999999,"15min":0.17000000000000001},"max":{"1min":0.41799999999999998,"5min":0.42099999999999999,"15min":0.42699999999999999},"last":0.28699999999999998},{"interface":"front","average":{"1min":0.29399999999999998,"5min":0.29099999999999998,"15min":0.27700000000000002},"min":{"1min":0.19900000000000001,"5min":0.182,"15min":0.16600000000000001},"max":{"1min":0.441,"5min":0.441,"15min":0.45800000000000002},"last":0.26300000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:38:19.718+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:46.404290+0000 front 2023-03-25T04:27:46.404350+0000 (oldest deadline 2023-03-25T04:28:11.704294+0000)
, 2023-03-25T04:38:20.315+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 since back 2023-03-25T04:27:45.086972+0000 front 2023-03-25T04:27:45.087017+0000 (oldest deadline 2023-03-25T04:28:10.986828+0000)

Ignoring malparsed backtrace: 2023-03-25T04:38:44.283 INFO:teuthology.orchestra.run.smithi070.stdout:
, 2023-03-25T04:38:44.283 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":781,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":16,"modified":"2023-03-25T04:38:40.225321+0000","services":{}},"progress_events":{}}
, 2023-03-25T04:38:44.296 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now

Ignoring malparsed backtrace: 2023-03-25T04:39:02.477+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:39:02.578+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:39:03.524+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:39:20.359 INFO:teuthology.orchestra.run.smithi070.stdout:
, 2023-03-25T04:39:20.359 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":818,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":16,"modified":"2023-03-25T04:38:40.225321+0000","services":{}},"progress_events":{}}
, 2023-03-25T04:39:20.372 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now

Ignoring malparsed backtrace: 2023-03-25T04:39:37.527 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:39:37.541 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":450,"stamp":"2023-03-25T04:39:36.235332+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002086"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":38,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:36:03.347070+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:36:03.347070+0000","last_peered":"2023-03-25T04:36:03.347070+0000","last_clean":"2023-03-25T04:36:03.347070+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:36:03.347070+0000","last_undegraded":"2023-03-25T04:36:03.347070+0000","last_fullsized":"2023-03-25T04:36:03.347070+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:36:03.347025+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:36:03.347025+0000","last_peered":"2023-03-25T04:36:03.347025+0000","last_clean":"2023-03-25T04:36:03.347025+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:36:03.347025+0000","last_undegraded":"2023-03-25T04:36:03.347025+0000","last_fullsized":"2023-03-25T04:36:03.347025+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:23.353778+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:37:23.353778+0000","last_peered":"2023-03-25T04:37:23.353778+0000","last_clean":"2023-03-25T04:37:23.353778+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:37:23.353778+0000","last_undegraded":"2023-03-25T04:37:23.353778+0000","last_fullsized":"2023-03-25T04:37:23.353778+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:35.587025+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:35:35.587025+0000","last_peered":"2023-03-25T04:35:35.587025+0000","last_clean":"2023-03-25T04:35:35.587025+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:35:35.587025+0000","last_undegraded":"2023-03-25T04:35:35.587025+0000","last_fullsized":"2023-03-25T04:35:35.587025+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":38,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:43.345308+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:35:43.345308+0000","last_peered":"2023-03-25T04:35:43.345308+0000","last_clean":"2023-03-25T04:35:43.345308+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:35:43.345308+0000","last_undegraded":"2023-03-25T04:35:43.345308+0000","last_fullsized":"2023-03-25T04:35:43.345308+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":30,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:35:43.345324+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:35:43.345324+0000","last_peered":"2023-03-25T04:35:43.345324+0000","last_clean":"2023-03-25T04:35:43.345324+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:35:43.345324+0000","last_undegraded":"2023-03-25T04:35:43.345324+0000","last_fullsized":"2023-03-25T04:35:43.345324+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117143,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:38:54 2023","interfaces":[{"interface":"back","average":{"1min":0.25800000000000001,"5min":0.25700000000000001,"15min":0.27400000000000002},"min":{"1min":0.192,"5min":0.17399999999999999,"15min":0.16},"max":{"1min":0.39400000000000002,"5min":0.44600000000000001,"15min":0.45100000000000001},"last":0.26500000000000001},{"interface":"front","average":{"1min":0.25900000000000001,"5min":0.26700000000000002,"15min":0.27900000000000003},"min":{"1min":0.19500000000000001,"5min":0.188,"15min":0.17499999999999999},"max":{"1min":0.41499999999999998,"5min":0.46400000000000002,"15min":0.46400000000000002},"last":0.29099999999999998}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051733,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:38:45 2023","interfaces":[{"interface":"back","average":{"1min":0.27300000000000002,"5min":0.28100000000000003,"15min":0.26400000000000001},"min":{"1min":0.19400000000000001,"5min":0.17299999999999999,"15min":0.17000000000000001},"max":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"last":0.27300000000000002},{"interface":"front","average":{"1min":0.28799999999999998,"5min":0.29099999999999998,"15min":0.27800000000000002},"min":{"1min":0.19800000000000001,"5min":0.182,"15min":0.16600000000000001},"max":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"last":0.308}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:39:54.416+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:39:54.455 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:39:54.780 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:40:09.288+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:40:09.389+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:40:10.267+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)

Ignoring malparsed backtrace: 2023-03-25T04:40:28.984 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":476,"stamp":"2023-03-25T04:40:28.245911+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002693"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":39,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:38:13.358968+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:38:13.358968+0000","last_peered":"2023-03-25T04:38:13.358968+0000","last_clean":"2023-03-25T04:38:13.358968+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:38:13.358968+0000","last_undegraded":"2023-03-25T04:38:13.358968+0000","last_fullsized":"2023-03-25T04:38:13.358968+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:38:13.358922+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:38:13.358922+0000","last_peered":"2023-03-25T04:38:13.358922+0000","last_clean":"2023-03-25T04:38:13.358922+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:38:13.358922+0000","last_undegraded":"2023-03-25T04:38:13.358922+0000","last_fullsized":"2023-03-25T04:38:13.358922+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:23.353778+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:37:23.353778+0000","last_peered":"2023-03-25T04:37:23.353778+0000","last_clean":"2023-03-25T04:37:23.353778+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:37:23.353778+0000","last_undegraded":"2023-03-25T04:37:23.353778+0000","last_fullsized":"2023-03-25T04:37:23.353778+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:40.598579+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:37:40.598579+0000","last_peered":"2023-03-25T04:37:40.598579+0000","last_clean":"2023-03-25T04:37:40.598579+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:37:40.598579+0000","last_undegraded":"2023-03-25T04:37:40.598579+0000","last_fullsized":"2023-03-25T04:37:40.598579+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":39,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:48.356344+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:37:48.356344+0000","last_peered":"2023-03-25T04:37:48.356344+0000","last_clean":"2023-03-25T04:37:48.356344+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:37:48.356344+0000","last_undegraded":"2023-03-25T04:37:48.356344+0000","last_fullsized":"2023-03-25T04:37:48.356344+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:48.356365+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:37:48.356365+0000","last_peered":"2023-03-25T04:37:48.356365+0000","last_clean":"2023-03-25T04:37:48.356365+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:37:48.356365+0000","last_undegraded":"2023-03-25T04:37:48.356365+0000","last_fullsized":"2023-03-25T04:37:48.356365+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117153,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:39:54 2023","interfaces":[{"interface":"back","average":{"1min":0.25900000000000001,"5min":0.25600000000000001,"15min":0.27100000000000002},"min":{"1min":0.188,"5min":0.17899999999999999,"15min":0.16},"max":{"1min":0.41999999999999998,"5min":0.44600000000000001,"15min":0.45100000000000001},"last":0.255},{"interface":"front","average":{"1min":0.27500000000000002,"5min":0.26600000000000001,"15min":0.27700000000000002},"min":{"1min":0.17699999999999999,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.45600000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"last":0.26800000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051743,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:39:47 2023","interfaces":[{"interface":"back","average":{"1min":0.26300000000000001,"5min":0.27700000000000002,"15min":0.26500000000000001},"min":{"1min":0.187,"5min":0.17299999999999999,"15min":0.17000000000000001},"max":{"1min":0.433,"5min":0.433,"15min":0.433},"last":0.36599999999999999},{"interface":"front","average":{"1min":0.27100000000000002,"5min":0.28799999999999998,"15min":0.27900000000000003},"min":{"1min":0.20100000000000001,"5min":0.182,"15min":0.16600000000000001},"max":{"1min":0.48599999999999999,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.34999999999999998}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:40:28.986 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean
, 2023-03-25T04:40:28.986 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}

Ignoring malparsed backtrace: 2023-03-25T04:40:47.229+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:40:47.503+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:40:48.251+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)

Ignoring malparsed backtrace: 2023-03-25T04:41:03.267 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:41:03.578 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all

Ignoring malparsed backtrace: 2023-03-25T04:41:21.347 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":502,"stamp":"2023-03-25T04:41:20.256204+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002342"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":39,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:38:13.358968+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:38:13.358968+0000","last_peered":"2023-03-25T04:38:13.358968+0000","last_clean":"2023-03-25T04:38:13.358968+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:38:13.358968+0000","last_undegraded":"2023-03-25T04:38:13.358968+0000","last_fullsized":"2023-03-25T04:38:13.358968+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:38:13.358922+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:38:13.358922+0000","last_peered":"2023-03-25T04:38:13.358922+0000","last_clean":"2023-03-25T04:38:13.358922+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:38:13.358922+0000","last_undegraded":"2023-03-25T04:38:13.358922+0000","last_fullsized":"2023-03-25T04:38:13.358922+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:23.353778+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:37:23.353778+0000","last_peered":"2023-03-25T04:37:23.353778+0000","last_clean":"2023-03-25T04:37:23.353778+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:37:23.353778+0000","last_undegraded":"2023-03-25T04:37:23.353778+0000","last_fullsized":"2023-03-25T04:37:23.353778+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:40.598579+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:37:40.598579+0000","last_peered":"2023-03-25T04:37:40.598579+0000","last_clean":"2023-03-25T04:37:40.598579+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:37:40.598579+0000","last_undegraded":"2023-03-25T04:37:40.598579+0000","last_fullsized":"2023-03-25T04:37:40.598579+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":39,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:48.356344+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:37:48.356344+0000","last_peered":"2023-03-25T04:37:48.356344+0000","last_clean":"2023-03-25T04:37:48.356344+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:37:48.356344+0000","last_undegraded":"2023-03-25T04:37:48.356344+0000","last_fullsized":"2023-03-25T04:37:48.356344+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:37:48.356365+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:37:48.356365+0000","last_peered":"2023-03-25T04:37:48.356365+0000","last_clean":"2023-03-25T04:37:48.356365+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:37:48.356365+0000","last_undegraded":"2023-03-25T04:37:48.356365+0000","last_fullsized":"2023-03-25T04:37:48.356365+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117163,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:40:56 2023","interfaces":[{"interface":"back","average":{"1min":0.28799999999999998,"5min":0.26300000000000001,"15min":0.27100000000000002},"min":{"1min":0.188,"5min":0.184,"15min":0.16},"max":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.17499999999999999},{"interface":"front","average":{"1min":0.29599999999999999,"5min":0.27300000000000002,"15min":0.27700000000000002},"min":{"1min":0.20599999999999999,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.43099999999999999,"5min":0.46400000000000002,"15min":0.46400000000000002},"last":0.20699999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051754,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:40:52 2023","interfaces":[{"interface":"back","average":{"1min":0.28599999999999998,"5min":0.27500000000000002,"15min":0.26700000000000002},"min":{"1min":0.17899999999999999,"5min":0.17899999999999999,"15min":0.17000000000000001},"max":{"1min":0.44,"5min":0.44,"15min":0.44},"last":0.23699999999999999},{"interface":"front","average":{"1min":0.29799999999999999,"5min":0.28699999999999998,"15min":0.28100000000000003},"min":{"1min":0.20300000000000001,"5min":0.182,"15min":0.16600000000000001},"max":{"1min":0.47599999999999998,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.28000000000000003}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:41:21.349 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean
, 2023-03-25T04:41:21.349 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}

Ignoring malparsed backtrace: 2023-03-25T04:41:39.867+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:41:40.383+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:41:40.905+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:41:58.219 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json
, 2023-03-25T04:41:58.319+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:41:58.644 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:42:18.193 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":530,"stamp":"2023-03-25T04:42:16.266659+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002356"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":39,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:38:13.358968+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:38:13.358968+0000","last_peered":"2023-03-25T04:38:13.358968+0000","last_clean":"2023-03-25T04:38:13.358968+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:38:13.358968+0000","last_undegraded":"2023-03-25T04:38:13.358968+0000","last_fullsized":"2023-03-25T04:38:13.358968+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":31,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:38:13.358922+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:38:13.358922+0000","last_peered":"2023-03-25T04:38:13.358922+0000","last_clean":"2023-03-25T04:38:13.358922+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:38:13.358922+0000","last_undegraded":"2023-03-25T04:38:13.358922+0000","last_fullsized":"2023-03-25T04:38:13.358922+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:23.364593+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:39:23.364593+0000","last_peered":"2023-03-25T04:39:23.364593+0000","last_clean":"2023-03-25T04:39:23.364593+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:39:23.364593+0000","last_undegraded":"2023-03-25T04:39:23.364593+0000","last_fullsized":"2023-03-25T04:39:23.364593+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:45.608475+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:39:45.608475+0000","last_peered":"2023-03-25T04:39:45.608475+0000","last_clean":"2023-03-25T04:39:45.608475+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:39:45.608475+0000","last_undegraded":"2023-03-25T04:39:45.608475+0000","last_fullsized":"2023-03-25T04:39:45.608475+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":40,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:53.367398+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:39:53.367398+0000","last_peered":"2023-03-25T04:39:53.367398+0000","last_clean":"2023-03-25T04:39:53.367398+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:39:53.367398+0000","last_undegraded":"2023-03-25T04:39:53.367398+0000","last_fullsized":"2023-03-25T04:39:53.367398+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:53.367424+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:39:53.367424+0000","last_peered":"2023-03-25T04:39:53.367424+0000","last_clean":"2023-03-25T04:39:53.367424+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:39:53.367424+0000","last_undegraded":"2023-03-25T04:39:53.367424+0000","last_fullsized":"2023-03-25T04:39:53.367424+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117175,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:41:58 2023","interfaces":[{"interface":"back","average":{"1min":0.25800000000000001,"5min":0.26400000000000001,"15min":0.26800000000000002},"min":{"1min":0.17499999999999999,"5min":0.17499999999999999,"15min":0.16},"max":{"1min":0.39100000000000001,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.22900000000000001},{"interface":"front","average":{"1min":0.26300000000000001,"5min":0.27200000000000002,"15min":0.27400000000000002},"min":{"1min":0.20599999999999999,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.35699999999999998,"5min":0.46400000000000002,"15min":0.46400000000000002},"last":0.26400000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051765,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:41:52 2023","interfaces":[{"interface":"back","average":{"1min":0.26500000000000001,"5min":0.27200000000000002,"15min":0.26800000000000002},"min":{"1min":0.17000000000000001,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.39200000000000002,"5min":0.44,"15min":0.44},"last":0.23899999999999999},{"interface":"front","average":{"1min":0.28100000000000003,"5min":0.28599999999999998,"15min":0.28199999999999997},"min":{"1min":0.19400000000000001,"5min":0.19400000000000001,"15min":0.16600000000000001},"max":{"1min":0.41399999999999998,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.254}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:42:18.297+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:42:18.958+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:42:35.512 INFO:teuthology.orchestra.run.smithi070.stdout:
, 2023-03-25T04:42:35.512 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":1013,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":17,"modified":"2023-03-25T04:41:54.262744+0000","services":{}},"progress_events":{}}
, 2023-03-25T04:42:35.525 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:42:52.741 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":548,"stamp":"2023-03-25T04:42:52.273601+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002221"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":40,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:40:23.370056+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:40:23.370056+0000","last_peered":"2023-03-25T04:40:23.370056+0000","last_clean":"2023-03-25T04:40:23.370056+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:40:23.370056+0000","last_undegraded":"2023-03-25T04:40:23.370056+0000","last_fullsized":"2023-03-25T04:40:23.370056+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:40:23.370039+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:40:23.370039+0000","last_peered":"2023-03-25T04:40:23.370039+0000","last_clean":"2023-03-25T04:40:23.370039+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:40:23.370039+0000","last_undegraded":"2023-03-25T04:40:23.370039+0000","last_fullsized":"2023-03-25T04:40:23.370039+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:23.364593+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:39:23.364593+0000","last_peered":"2023-03-25T04:39:23.364593+0000","last_clean":"2023-03-25T04:39:23.364593+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:39:23.364593+0000","last_undegraded":"2023-03-25T04:39:23.364593+0000","last_fullsized":"2023-03-25T04:39:23.364593+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:45.608475+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:39:45.608475+0000","last_peered":"2023-03-25T04:39:45.608475+0000","last_clean":"2023-03-25T04:39:45.608475+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:39:45.608475+0000","last_undegraded":"2023-03-25T04:39:45.608475+0000","last_fullsized":"2023-03-25T04:39:45.608475+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":40,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:53.367398+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:39:53.367398+0000","last_peered":"2023-03-25T04:39:53.367398+0000","last_clean":"2023-03-25T04:39:53.367398+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:39:53.367398+0000","last_undegraded":"2023-03-25T04:39:53.367398+0000","last_fullsized":"2023-03-25T04:39:53.367398+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:53.367424+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:39:53.367424+0000","last_peered":"2023-03-25T04:39:53.367424+0000","last_clean":"2023-03-25T04:39:53.367424+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:39:53.367424+0000","last_undegraded":"2023-03-25T04:39:53.367424+0000","last_fullsized":"2023-03-25T04:39:53.367424+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117182,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:41:58 2023","interfaces":[{"interface":"back","average":{"1min":0.25800000000000001,"5min":0.26400000000000001,"15min":0.26800000000000002},"min":{"1min":0.17499999999999999,"5min":0.17499999999999999,"15min":0.16},"max":{"1min":0.39100000000000001,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.19600000000000001},{"interface":"front","average":{"1min":0.26300000000000001,"5min":0.27200000000000002,"15min":0.27400000000000002},"min":{"1min":0.20599999999999999,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.35699999999999998,"5min":0.46400000000000002,"15min":0.46400000000000002},"last":0.219}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051773,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:41:52 2023","interfaces":[{"interface":"back","average":{"1min":0.26500000000000001,"5min":0.27200000000000002,"15min":0.26800000000000002},"min":{"1min":0.17000000000000001,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.39200000000000002,"5min":0.44,"15min":0.44},"last":0.19},{"interface":"front","average":{"1min":0.28100000000000003,"5min":0.28599999999999998,"15min":0.28199999999999997},"min":{"1min":0.19400000000000001,"5min":0.19400000000000001,"15min":0.16600000000000001},"max":{"1min":0.41399999999999998,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.17599999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:42:52.742 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean

Ignoring malparsed backtrace: 2023-03-25T04:43:09.973 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:43:09.986 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":556,"stamp":"2023-03-25T04:43:08.276556+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002359"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":40,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:40:23.370056+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:40:23.370056+0000","last_peered":"2023-03-25T04:40:23.370056+0000","last_clean":"2023-03-25T04:40:23.370056+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:40:23.370056+0000","last_undegraded":"2023-03-25T04:40:23.370056+0000","last_fullsized":"2023-03-25T04:40:23.370056+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:40:23.370039+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:40:23.370039+0000","last_peered":"2023-03-25T04:40:23.370039+0000","last_clean":"2023-03-25T04:40:23.370039+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:40:23.370039+0000","last_undegraded":"2023-03-25T04:40:23.370039+0000","last_fullsized":"2023-03-25T04:40:23.370039+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:23.364593+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:39:23.364593+0000","last_peered":"2023-03-25T04:39:23.364593+0000","last_clean":"2023-03-25T04:39:23.364593+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:39:23.364593+0000","last_undegraded":"2023-03-25T04:39:23.364593+0000","last_fullsized":"2023-03-25T04:39:23.364593+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:45.608475+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:39:45.608475+0000","last_peered":"2023-03-25T04:39:45.608475+0000","last_clean":"2023-03-25T04:39:45.608475+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:39:45.608475+0000","last_undegraded":"2023-03-25T04:39:45.608475+0000","last_fullsized":"2023-03-25T04:39:45.608475+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":40,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:53.367398+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:39:53.367398+0000","last_peered":"2023-03-25T04:39:53.367398+0000","last_clean":"2023-03-25T04:39:53.367398+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:39:53.367398+0000","last_undegraded":"2023-03-25T04:39:53.367398+0000","last_fullsized":"2023-03-25T04:39:53.367398+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:39:53.367424+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:39:53.367424+0000","last_peered":"2023-03-25T04:39:53.367424+0000","last_clean":"2023-03-25T04:39:53.367424+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:39:53.367424+0000","last_undegraded":"2023-03-25T04:39:53.367424+0000","last_fullsized":"2023-03-25T04:39:53.367424+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117185,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:43:00 2023","interfaces":[{"interface":"back","average":{"1min":0.27400000000000002,"5min":0.26700000000000002,"15min":0.26600000000000001},"min":{"1min":0.19600000000000001,"5min":0.17499999999999999,"15min":0.16},"max":{"1min":0.42999999999999999,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.27800000000000002},{"interface":"front","average":{"1min":0.28199999999999997,"5min":0.27500000000000002,"15min":0.27300000000000002},"min":{"1min":0.185,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.47199999999999998,"5min":0.47199999999999998,"15min":0.47199999999999998},"last":0.247}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051776,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:42:54 2023","interfaces":[{"interface":"back","average":{"1min":0.26300000000000001,"5min":0.27000000000000002,"15min":0.26900000000000002},"min":{"1min":0.17399999999999999,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.438,"5min":0.44,"15min":0.44},"last":0.20100000000000001},{"interface":"front","average":{"1min":0.27400000000000002,"5min":0.28199999999999997,"15min":0.28299999999999997},"min":{"1min":0.17599999999999999,"5min":0.17599999999999999,"15min":0.16600000000000001},"max":{"1min":0.41599999999999998,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.23499999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:43:25.042+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:43:25.411+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:43:26.013+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:43:47.937+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:43:48.534+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:43:48.889+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:44:09.318 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:44:09.331 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":586,"stamp":"2023-03-25T04:44:08.288933+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002697"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":40,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:40:23.370056+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:40:23.370056+0000","last_peered":"2023-03-25T04:40:23.370056+0000","last_clean":"2023-03-25T04:40:23.370056+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:40:23.370056+0000","last_undegraded":"2023-03-25T04:40:23.370056+0000","last_fullsized":"2023-03-25T04:40:23.370056+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":32,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:40:23.370039+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:40:23.370039+0000","last_peered":"2023-03-25T04:40:23.370039+0000","last_clean":"2023-03-25T04:40:23.370039+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:40:23.370039+0000","last_undegraded":"2023-03-25T04:40:23.370039+0000","last_fullsized":"2023-03-25T04:40:23.370039+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:23.375106+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:41:23.375106+0000","last_peered":"2023-03-25T04:41:23.375106+0000","last_clean":"2023-03-25T04:41:23.375106+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:41:23.375106+0000","last_undegraded":"2023-03-25T04:41:23.375106+0000","last_fullsized":"2023-03-25T04:41:23.375106+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:50.618930+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:41:50.618930+0000","last_peered":"2023-03-25T04:41:50.618930+0000","last_clean":"2023-03-25T04:41:50.618930+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:41:50.618930+0000","last_undegraded":"2023-03-25T04:41:50.618930+0000","last_fullsized":"2023-03-25T04:41:50.618930+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":41,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:58.377821+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:41:58.377821+0000","last_peered":"2023-03-25T04:41:58.377821+0000","last_clean":"2023-03-25T04:41:58.377821+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:41:58.377821+0000","last_undegraded":"2023-03-25T04:41:58.377821+0000","last_fullsized":"2023-03-25T04:41:58.377821+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:58.377836+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:41:58.377836+0000","last_peered":"2023-03-25T04:41:58.377836+0000","last_clean":"2023-03-25T04:41:58.377836+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:41:58.377836+0000","last_undegraded":"2023-03-25T04:41:58.377836+0000","last_fullsized":"2023-03-25T04:41:58.377836+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117197,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:44:01 2023","interfaces":[{"interface":"back","average":{"1min":0.27200000000000002,"5min":0.27000000000000002,"15min":0.26400000000000001},"min":{"1min":0.20399999999999999,"5min":0.17499999999999999,"15min":0.16},"max":{"1min":0.38400000000000001,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.245},{"interface":"front","average":{"1min":0.27600000000000002,"5min":0.27800000000000002,"15min":0.27100000000000002},"min":{"1min":0.23100000000000001,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.40400000000000003,"5min":0.47199999999999998,"15min":0.47199999999999998},"last":0.27500000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051788,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:43:55 2023","interfaces":[{"interface":"back","average":{"1min":0.26100000000000001,"5min":0.26700000000000002,"15min":0.27000000000000002},"min":{"1min":0.17799999999999999,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.374,"5min":0.44,"15min":0.44},"last":0.30299999999999999},{"interface":"front","average":{"1min":0.28299999999999997,"5min":0.28100000000000003,"15min":0.28399999999999997},"min":{"1min":0.20899999999999999,"5min":0.17599999999999999,"15min":0.16600000000000001},"max":{"1min":0.40100000000000002,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.27800000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:44:28.923 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":1126,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":19,"modified":"2023-03-25T04:43:20.278959+0000","services":{}},"progress_events":{}}
, 2023-03-25T04:44:28.936 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now
, 2023-03-25T04:44:28.937 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace: 2023-03-25T04:44:48.786 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:44:48.800 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":606,"stamp":"2023-03-25T04:44:48.296805+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002182"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":41,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:42:34.382196+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:42:34.382196+0000","last_peered":"2023-03-25T04:42:34.382196+0000","last_clean":"2023-03-25T04:42:34.382196+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:42:34.382196+0000","last_undegraded":"2023-03-25T04:42:34.382196+0000","last_fullsized":"2023-03-25T04:42:34.382196+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:42:29.381744+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:42:29.381744+0000","last_peered":"2023-03-25T04:42:29.381744+0000","last_clean":"2023-03-25T04:42:29.381744+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:42:29.381744+0000","last_undegraded":"2023-03-25T04:42:29.381744+0000","last_fullsized":"2023-03-25T04:42:29.381744+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:23.375106+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:41:23.375106+0000","last_peered":"2023-03-25T04:41:23.375106+0000","last_clean":"2023-03-25T04:41:23.375106+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:41:23.375106+0000","last_undegraded":"2023-03-25T04:41:23.375106+0000","last_fullsized":"2023-03-25T04:41:23.375106+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:50.618930+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:41:50.618930+0000","last_peered":"2023-03-25T04:41:50.618930+0000","last_clean":"2023-03-25T04:41:50.618930+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:41:50.618930+0000","last_undegraded":"2023-03-25T04:41:50.618930+0000","last_fullsized":"2023-03-25T04:41:50.618930+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":41,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:58.377821+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:41:58.377821+0000","last_peered":"2023-03-25T04:41:58.377821+0000","last_clean":"2023-03-25T04:41:58.377821+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:41:58.377821+0000","last_undegraded":"2023-03-25T04:41:58.377821+0000","last_fullsized":"2023-03-25T04:41:58.377821+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:58.377836+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:41:58.377836+0000","last_peered":"2023-03-25T04:41:58.377836+0000","last_clean":"2023-03-25T04:41:58.377836+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:41:58.377836+0000","last_undegraded":"2023-03-25T04:41:58.377836+0000","last_fullsized":"2023-03-25T04:41:58.377836+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117206,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:44:01 2023","interfaces":[{"interface":"back","average":{"1min":0.27200000000000002,"5min":0.27000000000000002,"15min":0.26400000000000001},"min":{"1min":0.20399999999999999,"5min":0.17499999999999999,"15min":0.16},"max":{"1min":0.38400000000000001,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.216},{"interface":"front","average":{"1min":0.27600000000000002,"5min":0.27800000000000002,"15min":0.27100000000000002},"min":{"1min":0.23100000000000001,"5min":0.17699999999999999,"15min":0.17499999999999999},"max":{"1min":0.40400000000000003,"5min":0.47199999999999998,"15min":0.47199999999999998},"last":0.23999999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051796,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:43:55 2023","interfaces":[{"interface":"back","average":{"1min":0.26100000000000001,"5min":0.26700000000000002,"15min":0.27000000000000002},"min":{"1min":0.17799999999999999,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.374,"5min":0.44,"15min":0.44},"last":0.25600000000000001},{"interface":"front","average":{"1min":0.28299999999999997,"5min":0.28100000000000003,"15min":0.28399999999999997},"min":{"1min":0.20899999999999999,"5min":0.17599999999999999,"15min":0.16600000000000001},"max":{"1min":0.40100000000000002,"5min":0.48599999999999999,"15min":0.48599999999999999},"last":0.28000000000000003}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:45:05.762 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:45:06.073 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all

Ignoring malparsed backtrace: 2023-03-25T04:45:25.658 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":624,"stamp":"2023-03-25T04:45:24.304294+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002366"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":41,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:42:34.382196+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:42:34.382196+0000","last_peered":"2023-03-25T04:42:34.382196+0000","last_clean":"2023-03-25T04:42:34.382196+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:42:34.382196+0000","last_undegraded":"2023-03-25T04:42:34.382196+0000","last_fullsized":"2023-03-25T04:42:34.382196+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:42:29.381744+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:42:29.381744+0000","last_peered":"2023-03-25T04:42:29.381744+0000","last_clean":"2023-03-25T04:42:29.381744+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:42:29.381744+0000","last_undegraded":"2023-03-25T04:42:29.381744+0000","last_fullsized":"2023-03-25T04:42:29.381744+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:23.375106+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:41:23.375106+0000","last_peered":"2023-03-25T04:41:23.375106+0000","last_clean":"2023-03-25T04:41:23.375106+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:41:23.375106+0000","last_undegraded":"2023-03-25T04:41:23.375106+0000","last_fullsized":"2023-03-25T04:41:23.375106+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:50.618930+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:41:50.618930+0000","last_peered":"2023-03-25T04:41:50.618930+0000","last_clean":"2023-03-25T04:41:50.618930+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:41:50.618930+0000","last_undegraded":"2023-03-25T04:41:50.618930+0000","last_fullsized":"2023-03-25T04:41:50.618930+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":41,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:58.377821+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:41:58.377821+0000","last_peered":"2023-03-25T04:41:58.377821+0000","last_clean":"2023-03-25T04:41:58.377821+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:41:58.377821+0000","last_undegraded":"2023-03-25T04:41:58.377821+0000","last_fullsized":"2023-03-25T04:41:58.377821+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":33,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:41:58.377836+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:41:58.377836+0000","last_peered":"2023-03-25T04:41:58.377836+0000","last_clean":"2023-03-25T04:41:58.377836+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:41:58.377836+0000","last_undegraded":"2023-03-25T04:41:58.377836+0000","last_fullsized":"2023-03-25T04:41:58.377836+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117213,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:45:01 2023","interfaces":[{"interface":"back","average":{"1min":0.251,"5min":0.26800000000000002,"15min":0.26400000000000001},"min":{"1min":0.183,"5min":0.17499999999999999,"15min":0.16},"max":{"1min":0.41499999999999998,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.26900000000000002},{"interface":"front","average":{"1min":0.28000000000000003,"5min":0.27900000000000003,"15min":0.27100000000000002},"min":{"1min":0.20999999999999999,"5min":0.185,"15min":0.17499999999999999},"max":{"1min":0.44400000000000001,"5min":0.47199999999999998,"15min":0.47199999999999998},"last":0.27900000000000003}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051803,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:44:55 2023","interfaces":[{"interface":"back","average":{"1min":0.252,"5min":0.26500000000000001,"15min":0.26900000000000002},"min":{"1min":0.183,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.39700000000000002,"5min":0.44,"15min":0.44},"last":0.20699999999999999},{"interface":"front","average":{"1min":0.27000000000000002,"5min":0.28100000000000003,"15min":0.28299999999999997},"min":{"1min":0.20300000000000001,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":0.42099999999999999,"5min":0.47599999999999998,"15min":0.48599999999999999},"last":0.23599999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:45:25.659 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json
, 2023-03-25T04:45:25.969 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: 2023-03-25T04:45:42.735+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:45:42.744+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:45:42.897 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace: 2023-03-25T04:46:00.183 INFO:teuthology.orchestra.run.smithi070.stdout:
, 2023-03-25T04:46:00.183 INFO:teuthology.orchestra.run.smithi070.stdout:{"fsid":"98c57530-5ed4-41ea-98ab-f0f17883425e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":1217,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":39,"num_osds":3,"num_up_osds":3,"osd_up_since":1679718463,"num_in_osds":3,"osd_in_since":1679718446,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18366464,"bytes_avail":289891926016,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":20,"modified":"2023-03-25T04:45:42.307700+0000","services":{"mgr":{"daemons":{"summary":"","x":{"start_epoch":0,"start_stamp":"0.000000","gid":0,"addr":"(unrecognized address family 0)/0","metadata":{},"task_status":{}}}}}},"progress_events":{}}
, 2023-03-25T04:46:00.197 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now

Ignoring malparsed backtrace: 2023-03-25T04:46:19.910+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:46:20.039 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all

Ignoring malparsed backtrace: 2023-03-25T04:46:39.611 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "32'101", 'reported_seq': 131, 'reported_epoch': 38, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-25T04:27:44.312768+0000', 'last_change': '2023-03-25T04:27:44.312768+0000', 'last_active': '2023-03-25T04:27:44.312768+0000', 'last_peered': '2023-03-25T04:27:44.312768+0000', 'last_clean': '2023-03-25T04:26:45.819843+0000', 'last_became_active': '2023-03-25T04:27:44.311757+0000', 'last_became_peered': '2023-03-25T04:27:44.311757+0000', 'last_unstale': '2023-03-25T04:27:44.312768+0000', 'last_undegraded': '2023-03-25T04:27:44.308893+0000', 'last_fullsized': '2023-03-25T04:27:44.312768+0000', 'mapping_epoch': 37, 'log_start': "20'95", 'ondisk_log_start': "20'95", 'created': 18, 'last_epoch_clean': 19, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'last_clean_scrub_stamp': '2023-03-25T04:26:11.322637+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []}
, 2023-03-25T04:46:39.611 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json
, 2023-03-25T04:46:39.613+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:46:58.485+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:46:58.764+0000 7f40a0541700 -1 osd.1 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:26.956396+0000 (oldest deadline 2023-03-25T04:38:46.956396+0000)
, 2023-03-25T04:46:59.519+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)

Ignoring malparsed backtrace: 2023-03-25T04:47:14.132 INFO:teuthology.orchestra.run.smithi070.stdout:
, dumped all
, 2023-03-25T04:47:14.146 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":678,"stamp":"2023-03-25T04:47:12.325044+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002571"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":42,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:44.393426+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:44:44.393426+0000","last_peered":"2023-03-25T04:44:44.393426+0000","last_clean":"2023-03-25T04:44:44.393426+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:44:44.393426+0000","last_undegraded":"2023-03-25T04:44:44.393426+0000","last_fullsized":"2023-03-25T04:44:44.393426+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:39.393044+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:44:39.393044+0000","last_peered":"2023-03-25T04:44:39.393044+0000","last_clean":"2023-03-25T04:44:39.393044+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:44:39.393044+0000","last_undegraded":"2023-03-25T04:44:39.393044+0000","last_fullsized":"2023-03-25T04:44:39.393044+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:43:24.386177+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:43:24.386177+0000","last_peered":"2023-03-25T04:43:24.386177+0000","last_clean":"2023-03-25T04:43:24.386177+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:43:24.386177+0000","last_undegraded":"2023-03-25T04:43:24.386177+0000","last_fullsized":"2023-03-25T04:43:24.386177+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:43:55.629798+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:43:55.629798+0000","last_peered":"2023-03-25T04:43:55.629798+0000","last_clean":"2023-03-25T04:43:55.629798+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:43:55.629798+0000","last_undegraded":"2023-03-25T04:43:55.629798+0000","last_fullsized":"2023-03-25T04:43:55.629798+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":42,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:04.389904+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:44:04.389904+0000","last_peered":"2023-03-25T04:44:04.389904+0000","last_clean":"2023-03-25T04:44:04.389904+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:44:04.389904+0000","last_undegraded":"2023-03-25T04:44:04.389904+0000","last_fullsized":"2023-03-25T04:44:04.389904+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:04.389923+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:44:04.389923+0000","last_peered":"2023-03-25T04:44:04.389923+0000","last_clean":"2023-03-25T04:44:04.389923+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:44:04.389923+0000","last_undegraded":"2023-03-25T04:44:04.389923+0000","last_fullsized":"2023-03-25T04:44:04.389923+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117235,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:47:05 2023","interfaces":[{"interface":"back","average":{"1min":0.27800000000000002,"5min":0.27000000000000002,"15min":0.26300000000000001},"min":{"1min":0.20100000000000001,"5min":0.16900000000000001,"15min":0.16},"max":{"1min":0.39200000000000002,"5min":0.42999999999999999,"15min":0.45600000000000002},"last":0.379},{"interface":"front","average":{"1min":0.28399999999999997,"5min":0.28100000000000003,"15min":0.27200000000000002},"min":{"1min":0.17799999999999999,"5min":0.17799999999999999,"15min":0.17699999999999999},"max":{"1min":0.42599999999999999,"5min":0.47199999999999998,"15min":0.47199999999999998},"last":0.39800000000000002}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051825,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:47:01 2023","interfaces":[{"interface":"back","average":{"1min":0.27300000000000002,"5min":0.26100000000000001,"15min":0.27000000000000002},"min":{"1min":0.186,"5min":0.17399999999999999,"15min":0.17000000000000001},"max":{"1min":0.40400000000000003,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.23899999999999999},{"interface":"front","average":{"1min":0.28699999999999998,"5min":0.27500000000000002,"15min":0.28299999999999997},"min":{"1min":0.186,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.48599999999999999},"last":0.19600000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}

Ignoring malparsed backtrace: 2023-03-25T04:47:31.198 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json
, 2023-03-25T04:47:31.411+0000 7f98f561b700 -1 osd.0 39 heartbeat_check: no reply from 172.21.15.70:6810 osd.2 ever on either front or back, first ping sent 2023-03-25T04:38:34.741548+0000 (oldest deadline 2023-03-25T04:38:54.741548+0000)
, 2023-03-25T04:47:31.789 INFO:teuthology.orchestra.run.smithi070.stdout:

Ignoring malparsed backtrace: dumped all
, 2023-03-25T04:47:49.509 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":696,"stamp":"2023-03-25T04:47:48.332093+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17936,"kb_used_data":1720,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097584,"statfs":{"total":289910292480,"available":289891926016,"internally_reserved":0,"allocated":1761280,"data_stored":389178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002697"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":42,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:44.393426+0000","last_change":"2023-03-25T04:27:45.314661+0000","last_active":"2023-03-25T04:44:44.393426+0000","last_peered":"2023-03-25T04:44:44.393426+0000","last_clean":"2023-03-25T04:44:44.393426+0000","last_became_active":"2023-03-25T04:27:45.314352+0000","last_became_peered":"2023-03-25T04:27:45.314352+0000","last_unstale":"2023-03-25T04:44:44.393426+0000","last_undegraded":"2023-03-25T04:44:44.393426+0000","last_fullsized":"2023-03-25T04:44:44.393426+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T14:58:05.796465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:39.393044+0000","last_change":"2023-03-25T04:27:28.611060+0000","last_active":"2023-03-25T04:44:39.393044+0000","last_peered":"2023-03-25T04:44:39.393044+0000","last_clean":"2023-03-25T04:44:39.393044+0000","last_became_active":"2023-03-25T04:27:28.610787+0000","last_became_peered":"2023-03-25T04:27:28.610787+0000","last_unstale":"2023-03-25T04:44:39.393044+0000","last_undegraded":"2023-03-25T04:44:39.393044+0000","last_fullsized":"2023-03-25T04:44:39.393044+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T11:31:48.515033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315194+0000","last_change":"2023-03-25T04:27:44.314479+0000","last_active":"2023-03-25T04:27:44.315194+0000","last_peered":"2023-03-25T04:27:44.315194+0000","last_clean":"2023-03-25T04:27:44.315194+0000","last_became_active":"2023-03-25T04:27:44.314204+0000","last_became_peered":"2023-03-25T04:27:44.314204+0000","last_unstale":"2023-03-25T04:27:44.315194+0000","last_undegraded":"2023-03-25T04:27:44.315194+0000","last_fullsized":"2023-03-25T04:27:44.315194+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T05:34:49.704283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:45:24.396717+0000","last_change":"2023-03-25T04:27:28.610329+0000","last_active":"2023-03-25T04:45:24.396717+0000","last_peered":"2023-03-25T04:45:24.396717+0000","last_clean":"2023-03-25T04:45:24.396717+0000","last_became_active":"2023-03-25T04:27:28.610046+0000","last_became_peered":"2023-03-25T04:27:28.610046+0000","last_unstale":"2023-03-25T04:45:24.396717+0000","last_undegraded":"2023-03-25T04:45:24.396717+0000","last_fullsized":"2023-03-25T04:45:24.396717+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:19:31.582028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":35,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:43:55.629798+0000","last_change":"2023-03-25T04:27:27.296154+0000","last_active":"2023-03-25T04:43:55.629798+0000","last_peered":"2023-03-25T04:43:55.629798+0000","last_clean":"2023-03-25T04:43:55.629798+0000","last_became_active":"2023-03-25T04:27:27.295819+0000","last_became_peered":"2023-03-25T04:27:27.295819+0000","last_unstale":"2023-03-25T04:43:55.629798+0000","last_undegraded":"2023-03-25T04:43:55.629798+0000","last_fullsized":"2023-03-25T04:43:55.629798+0000","mapping_epoch":31,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":32,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:20.704421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"32'101","reported_seq":131,"reported_epoch":38,"state":"active+recovering+degraded","last_fresh":"2023-03-25T04:27:44.312768+0000","last_change":"2023-03-25T04:27:44.312768+0000","last_active":"2023-03-25T04:27:44.312768+0000","last_peered":"2023-03-25T04:27:44.312768+0000","last_clean":"2023-03-25T04:26:45.819843+0000","last_became_active":"2023-03-25T04:27:44.311757+0000","last_became_peered":"2023-03-25T04:27:44.311757+0000","last_unstale":"2023-03-25T04:27:44.312768+0000","last_undegraded":"2023-03-25T04:27:44.308893+0000","last_fullsized":"2023-03-25T04:27:44.312768+0000","mapping_epoch":37,"log_start":"20'95","ondisk_log_start":"20'95","created":18,"last_epoch_clean":19,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:26:11.322637+0000","last_clean_scrub_stamp":"2023-03-25T04:26:11.322637+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:12:55.870961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":42,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:04.389904+0000","last_change":"2023-03-25T04:27:45.315064+0000","last_active":"2023-03-25T04:44:04.389904+0000","last_peered":"2023-03-25T04:44:04.389904+0000","last_clean":"2023-03-25T04:44:04.389904+0000","last_became_active":"2023-03-25T04:27:45.314753+0000","last_became_peered":"2023-03-25T04:27:45.314753+0000","last_unstale":"2023-03-25T04:44:04.389904+0000","last_undegraded":"2023-03-25T04:44:04.389904+0000","last_fullsized":"2023-03-25T04:44:04.389904+0000","mapping_epoch":38,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":39,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T15:29:57.411987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":34,"reported_epoch":39,"state":"active+clean","last_fresh":"2023-03-25T04:44:04.389923+0000","last_change":"2023-03-25T04:27:28.610420+0000","last_active":"2023-03-25T04:44:04.389923+0000","last_peered":"2023-03-25T04:44:04.389923+0000","last_clean":"2023-03-25T04:44:04.389923+0000","last_became_active":"2023-03-25T04:27:28.610150+0000","last_became_peered":"2023-03-25T04:27:28.610150+0000","last_unstale":"2023-03-25T04:44:04.389923+0000","last_undegraded":"2023-03-25T04:44:04.389923+0000","last_fullsized":"2023-03-25T04:44:04.389923+0000","mapping_epoch":32,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":33,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T08:38:30.315685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":38,"state":"active+clean","last_fresh":"2023-03-25T04:27:44.315182+0000","last_change":"2023-03-25T04:27:44.314475+0000","last_active":"2023-03-25T04:27:44.315182+0000","last_peered":"2023-03-25T04:27:44.315182+0000","last_clean":"2023-03-25T04:27:44.315182+0000","last_became_active":"2023-03-25T04:27:44.314229+0000","last_became_peered":"2023-03-25T04:27:44.314229+0000","last_unstale":"2023-03-25T04:27:44.315182+0000","last_undegraded":"2023-03-25T04:27:44.315182+0000","last_fullsized":"2023-03-25T04:27:44.315182+0000","mapping_epoch":37,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":38,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-25T04:25:58.301162+0000","last_clean_scrub_stamp":"2023-03-25T04:25:58.301162+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-26T13:48:17.382991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":37,"seq":158913789954,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365780,"statfs":{"total":96636764160,"available":96630558720,"internally_reserved":0,"allocated":692224,"data_stored":142172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23799999999999999}]},{"osd":1,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.252}]}]},{"osd":0,"up_from":27,"seq":115964117242,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5768,"kb_used_data":320,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366072,"statfs":{"total":96636764160,"available":96630857728,"internally_reserved":0,"allocated":327680,"data_stored":99288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Mar 25 04:47:05 2023","interfaces":[{"interface":"back","average":{"1min":0.27800000000000002,"5min":0.27000000000000002,"15min":0.26300000000000001},"min":{"1min":0.20100000000000001,"5min":0.16900000000000001,"15min":0.16},"max":{"1min":0.39200000000000002,"5min":0.42999999999999999,"15min":0.45600000000000002},"last":0.40500000000000003},{"interface":"front","average":{"1min":0.28399999999999997,"5min":0.28100000000000003,"15min":0.27200000000000002},"min":{"1min":0.17799999999999999,"5min":0.17799999999999999,"15min":0.17699999999999999},"max":{"1min":0.42599999999999999,"5min":0.47199999999999998,"15min":0.47199999999999998},"last":0.42399999999999999}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]}]},{"osd":1,"up_from":29,"seq":124554051834,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6108,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365732,"statfs":{"total":96636764160,"available":96630509568,"internally_reserved":0,"allocated":741376,"data_stored":147718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Mar 25 04:47:01 2023","interfaces":[{"interface":"back","average":{"1min":0.27300000000000002,"5min":0.26100000000000001,"15min":0.27000000000000002},"min":{"1min":0.186,"5min":0.17399999999999999,"15min":0.17000000000000001},"max":{"1min":0.40400000000000003,"5min":0.45600000000000002,"15min":0.45600000000000002},"last":0.25600000000000001},{"interface":"front","average":{"1min":0.28699999999999998,"5min":0.27500000000000002,"15min":0.28299999999999997},"min":{"1min":0.186,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.48599999999999999},"last":0.22800000000000001}]},{"osd":2,"last update":"Thu Jan  1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}}
, 2023-03-25T04:47:49.510 DEBUG:teuthology.orchestra.run.smithi070:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json

Ignoring malparsed backtrace:     ctx.managers[config['cluster']].wait_for_clean()
,   File "/home/teuthworker/src/github.com_ceph_ceph-c_511015c23eef0684a64171b710e432448df2b7c7/qa/tasks/ceph_manager.py", line 2747, in wait_for_clean
,     assert time.time() - start < timeout, \

Found 31 distinct failure reasons
Dead: b'2023-03-25T20:33:13.050 DEBUG:teuthology.exit:Finished running handlers'
56 jobs
suites intersection: ['rados']
suites union: ['0-size-min-size-overrides/2-size-2-min-size', '0-size-min-size-overrides/3-size-2-min-size', '1-install/nautilus', '1-install/nautilus-v2only', '1-install/octopus', '1-install/quincy', '1-pg-log-overrides/normal_pg_log', '1-pg-log-overrides/short_pg_log', '2-recovery-overrides/{default}', '2-recovery-overrides/{more-active-recovery}', '2-recovery-overrides/{more-async-partial-recovery}', '2-recovery-overrides/{more-async-recovery}', '2-recovery-overrides/{more-partial-recovery}', '3-scrub-overrides/{default}', '3-scrub-overrides/{max-simultaneous-scrubs-2}', '3-scrub-overrides/{max-simultaneous-scrubs-3}', 'backoff/normal', 'backoff/peering', 'backoff/peering_and_degraded', 'ceph', 'cluster/{12-osds', 'clusters/{fixed-2', 'clusters/{fixed-4', 'clusters/{openstack', 'crc-failures/bad_map_crc_failure', 'crc-failures/default', 'd-balancer/crush-compat', 'd-balancer/on', 'fast/fast', 'fast/normal', 'mon_election/classic', 'mon_election/connectivity', 'msgr-failures/fastclose', 'msgr-failures/few', 'msgr-failures/osd-delay', 'msgr-failures/osd-dispatch-delay', 'msgr/async', 'msgr/async-v1only', 'msgr/async-v2only', 'objectstore/bluestore-bitmap', 'objectstore/bluestore-comp-lz4', 'objectstore/bluestore-comp-snappy', 'objectstore/bluestore-comp-zlib', 'objectstore/bluestore-comp-zstd', 'objectstore/bluestore-hybrid', 'objectstore/bluestore-low-osd-mem-target', 'objectstore/bluestore-stupid', 'openstack}', 'rados', 'rados/singleton/{all/thrash-backfill-full', 'rados/thrash-erasure-code-big/{ceph', 'rados/thrash-erasure-code-isa/{arch/x86_64', 'rados/thrash-erasure-code-overwrites/{bluestore-bitmap', 'rados/thrash-erasure-code-shec/{ceph', 'rados/thrash-erasure-code/{ceph', 'rados/thrash-old-clients/{0-distro$/{centos_8.stream_container_tools}', 'rados/thrash/{0-size-min-size-overrides/2-size-2-min-size', 'rados/thrash/{0-size-min-size-overrides/3-size-2-min-size', 'recovery-overrides/{default}', 'recovery-overrides/{more-active-recovery}', 'recovery-overrides/{more-async-partial-recovery}', 'recovery-overrides/{more-async-recovery}', 'recovery-overrides/{more-partial-recovery}', 'supported-random-distro$/{centos_8}', 'supported-random-distro$/{centos_8}}', 'supported-random-distro$/{rhel_8}', 'supported-random-distro$/{ubuntu_latest}', 'thrashers/careful', 'thrashers/default', 'thrashers/fastread', 'thrashers/mapgap', 'thrashers/minsize_recovery', 'thrashers/morepggrow', 'thrashers/none', 'thrashers/pggrow', 'thrashosds-health', 'three-plus-one}', 'workloads/cache-agent-small}', 'workloads/cache-pool-snaps-readproxy}', 'workloads/cache-pool-snaps}', 'workloads/cache-snaps}', 'workloads/cache}', 'workloads/ec-pool-snaps-few-objects-overwrites}', 'workloads/ec-rados-plugin=clay-k=4-m=2}', 'workloads/ec-rados-plugin=isa-k=2-m=1}', 'workloads/ec-rados-plugin=jerasure-k=2-m=1}', 'workloads/ec-rados-plugin=jerasure-k=3-m=1}', 'workloads/ec-rados-plugin=jerasure-k=4-m=2}', 'workloads/ec-rados-plugin=lrc-k=4-m=2-l=3}', 'workloads/ec-rados-plugin=shec-k=4-m=3-c=2}', 'workloads/ec-small-objects-fast-read-overwrites}', 'workloads/ec-small-objects-fast-read}', 'workloads/ec-small-objects-overwrites}', 'workloads/ec-small-objects}', 'workloads/ec-snaps-few-objects-overwrites}', 'workloads/pool-snaps-few-objects}', 'workloads/radosbench}', 'workloads/redirect_set_object}', 'workloads/set-chunks-read}', 'workloads/small-objects-balanced}', 'workloads/small-objects-localized}', 'workloads/small-objects}', 'workloads/snaps-few-objects-balanced}', 'workloads/snaps-few-objects-localized}', 'workloads/snaps-few-objects}', 'workloads/write_fadvise_dontneed}']

Dead: b"TypeError: argument of type 'NoneType' is not iterable"
1 jobs: ['7211028']
suites: ['mon_election/connectivity', 'msgr-failures/none', 'msgr/async-v2only', 'objectstore/bluestore-comp-zstd', 'rados', 'rados/singleton/{all/max-pg-per-osd.from-mon', 'supported-random-distro$/{rhel_8}}']

Failure: 'pg2json'
1 jobs: ['7210931']
suites: ['rados/objectstore/{backends/ceph_objectstore_tool', 'supported-random-distro$/{centos_8}}']

Timeout 3h running clone.client.0/qa/workunits/cls/test_cls_2pc_queue.sh
2 jobs: ['7211090', '7210886']
suites intersection: ['ceph', 'clusters/{fixed-2', 'd-thrash/default/{default', 'mon_election/connectivity', 'msgr-failures/few', 'msgr/async-v2only', 'openstack}', 'rados', 'rados/verify/{centos_latest', 'tasks/rados_cls_all', 'thrashosds-health}', 'validater/valgrind}']
suites union: ['ceph', 'clusters/{fixed-2', 'd-thrash/default/{default', 'mon_election/connectivity', 'msgr-failures/few', 'msgr/async-v2only', 'objectstore/bluestore-comp-lz4', 'objectstore/bluestore-comp-zlib', 'openstack}', 'rados', 'rados/verify/{centos_latest', 'tasks/rados_cls_all', 'thrashosds-health}', 'validater/valgrind}']

Failure:  ++ sudo adjust-ulimits ceph-objectstore-tool --data-path /var/lib/ceph/osd/ceph-1 --journal-path /var/lib/ceph/osd/ceph-1/journal --pgid 11.0s0 --op list
++ grep '"oid":"repair_test_obj"'
*** Caught signal (Segmentation fault) **
 in thread 7f0a8b7c4440 thread_name:ceph-objectstor
 ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
 1: /lib64/libpthread.so.0(+0x12cf0) [0x7f0a88840cf0]
 2: __pthread_mutex_lock()
 3: (BlueFS::dir_exists(std::basic_string_view<char, std::char_traits<char> >)+0x46) [0x560f6f421ec6]
 4: (BlueStore::restore_snap_mapper(GlobalSnapMapper&)+0x4a) [0x560f6f3a3d9a]
 5: main()
 6: __libc_start_main()
 7: _start()
+ object=

4 jobs: ['7211120', '7210845', '7210851', '7210991']
suites intersection: ['mon_election/classic', 'msgr-failures/few', 'rados']
suites union: ['clusters/{fixed-2', 'mon_election/classic', 'msgr-failures/few', 'msgr/async', 'msgr/async-v1only', 'objectstore/bluestore-bitmap', 'objectstore/bluestore-comp-zstd', 'objectstore/bluestore-hybrid', 'objectstore/bluestore-stupid', 'openstack}', 'rados', 'rados/basic/{ceph', 'rados/singleton/{all/ec-inconsistent-hinfo', 'rados/singleton/{all/rebuild-mondb', 'supported-random-distro$/{centos_8}', 'supported-random-distro$/{rhel_8}}', 'supported-random-distro$/{ubuntu_latest}', 'tasks/repair_test}']

Timeout 6h running clone.client.0/qa/workunits/rados/test.sh
3 jobs: ['7211056', '7210957', '7210853']
suites intersection: ['ceph', 'clusters/{fixed-2', 'msgr-failures/few', 'msgr/async-v1only', 'openstack}', 'rados', 'rados/verify/{centos_latest', 'tasks/rados_api_tests']
suites union: ['ceph', 'clusters/{fixed-2', 'd-thrash/default/{default', 'd-thrash/none', 'mon_election/classic', 'mon_election/connectivity', 'msgr-failures/few', 'msgr/async-v1only', 'objectstore/bluestore-bitmap', 'objectstore/bluestore-comp-snappy', 'objectstore/bluestore-hybrid', 'openstack}', 'rados', 'rados/verify/{centos_latest', 'tasks/rados_api_tests', 'thrashosds-health}', 'validater/lockdep}', 'validater/valgrind}']

Failure: Command failed on smithi064 with status 1: 'sudo systemctl enable --now kubelet && sudo kubeadm config images pull'
4 jobs: ['7210856', '7211007', '7211079', '7210929']
suites intersection: ['0-kubeadm', '0-nvme-loop', '1-rook', 'k8s/1.21', 'rados/rook/smoke/{0-distro/ubuntu_20.04']
suites union: ['0-kubeadm', '0-nvme-loop', '1-rook', '2-workload/none', '2-workload/radosbench', 'cluster/1-node', 'cluster/3-node', 'k8s/1.21', 'net/calico', 'net/flannel', 'net/host', 'rados/rook/smoke/{0-distro/ubuntu_20.04', 'rook/1.7.2}', 'rook/master}']

Failure: Command failed (workunit test cephtool/test_kvstore_tool.sh) on smithi129 with status 139: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=511015c23eef0684a64171b710e432448df2b7c7 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cephtool/test_kvstore_tool.sh'
6 jobs: ['7211045', '7211116', '7210903', '7210959', '7210976', '7211106']
suites intersection: ['rados']
suites union: ['mon_election/classic', 'mon_election/connectivity', 'msgr-failures/few', 'msgr-failures/many', 'msgr-failures/none', 'msgr/async', 'msgr/async-v1only', 'msgr/async-v2only', 'objectstore/bluestore-bitmap', 'objectstore/bluestore-comp-lz4', 'objectstore/bluestore-comp-snappy', 'rados', 'rados/singleton-bluestore/{all/cephtool', 'rados/singleton-nomsgr/{all/ceph-kvstore-tool', 'supported-random-distro$/{centos_8}}', 'supported-random-distro$/{rhel_8}}', 'supported-random-distro$/{ubuntu_latest}}']

Timeout 3h running clone.client.0/qa/workunits/rados/load-gen-mix-small.sh
1 jobs: ['7210912']
suites: ['mon_election/classic', 'msgr-failures/none', 'msgr/async-v2only', 'objectstore/bluestore-hybrid', 'rados', 'rados/singleton/{all/thrash-rados/{thrash-rados', 'supported-random-distro$/{ubuntu_latest}}', 'thrashosds-health}']

Failure: Command failed (workunit test rados/test_python.sh) on smithi078 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=511015c23eef0684a64171b710e432448df2b7c7 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 1h /home/ubuntu/cephtest/clone.client.0/qa/workunits/rados/test_python.sh'
1 jobs: ['7210961']
suites: ['clusters/{fixed-2', 'mon_election/connectivity', 'msgr-failures/many', 'msgr/async', 'objectstore/bluestore-hybrid', 'openstack}', 'rados', 'rados/basic/{ceph', 'supported-random-distro$/{rhel_8}', 'tasks/rados_python}']

Timeout 120 running ceph --cluster ceph osd pool create unique_pool_0 16
1 jobs: ['7211129']
suites: ['1-pg-log-overrides/short_pg_log', '2-recovery-overrides/{more-partial-recovery}', '3-scrub-overrides/{default}', 'backoff/peering_and_degraded', 'ceph', 'clusters/{fixed-2', 'crc-failures/default', 'd-balancer/on', 'mon_election/connectivity', 'msgr-failures/few', 'msgr/async', 'objectstore/bluestore-stupid', 'openstack}', 'rados', 'rados/thrash/{0-size-min-size-overrides/3-size-2-min-size', 'supported-random-distro$/{ubuntu_latest}', 'thrashers/pggrow', 'thrashosds-health', 'workloads/dedup-io-mixed}']

Timeout 3h running clone.client.0/qa/workunits/rbd/test_librbd.sh
1 jobs: ['7210935']
suites: ['0-start', '1-tasks', 'mon_election/classic', 'rados/upgrade/parallel/{0-random-distro$/{rhel_8.6_container_tools_3.0}', 'rados_api', 'rados_loadgenbig', 'rbd_import_export', 'test_rbd_api', 'test_rbd_python}}', 'upgrade-sequence', 'workload/{ec-rados-default']

Failure: Command failed (workunit test osd-backfill/osd-backfill-recovery-log.sh) on smithi184 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=511015c23eef0684a64171b710e432448df2b7c7 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/standalone/osd-backfill/osd-backfill-recovery-log.sh'
1 jobs: ['7211132']
suites: ['rados/standalone/{supported-random-distro$/{rhel_8}', 'workloads/osd-backfill}']

Failure: Command failed (workunit test misc/test-ceph-helpers.sh) on smithi177 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=511015c23eef0684a64171b710e432448df2b7c7 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/standalone/misc/test-ceph-helpers.sh'
1 jobs: ['7211066']
suites: ['rados/standalone/{supported-random-distro$/{rhel_8}', 'workloads/misc}']

Crash: Command failed on smithi027 with status 1: 'CEPH_CLIENT_ID=0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph_test_rados --set_chunk --enable_dedup --low_tier_pool low_tier --dedup_chunk_size 131072 --dedup_chunk_algo fastcdc --max-ops 1500 --objects 50 --max-in-flight 16 --size 4000000 --min-stride-size 400000 --max-stride-size 800000 --max-seconds 0 --op read 100 --op write 25 --op delete 10 --op snap_create 10 --op snap_remove 10 --op rollback 10 --op set_chunk 30 --op tier_promote 10 --op tier_evict 10 --op tier_flush 5 --op write_excl 25 --pool unique_pool_0'
ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
 1: /lib/x86_64-linux-gnu/libpthread.so.0(+0x14420) [0x7fbabf399420]
 2: pthread_mutex_lock()
 3: (BlueFS::dir_exists(std::basic_string_view<char, std::char_traits<char> >)+0x46) [0x55d455f7a4b6]
 4: (BlueStore::restore_snap_mapper(GlobalSnapMapper&)+0x4a) [0x55d455eda3aa]
 5: main()
 6: __libc_start_main()
 7: _start()
9 jobs: ['7210872', '7210910', '7211023', '7211042', '7211060', '7211093', '7210904', '7211087', '7211020']
suites intersection: ['openstack}', 'rados']
suites union: ['1-pg-log-overrides/normal_pg_log', '1-pg-log-overrides/short_pg_log', '2-recovery-overrides/{default}', '2-recovery-overrides/{more-active-recovery}', '2-recovery-overrides/{more-partial-recovery}', '3-scrub-overrides/{default}', '3-scrub-overrides/{max-simultaneous-scrubs-3}', 'backoff/normal', 'backoff/peering_and_degraded', 'ceph', 'clusters/{fixed-2', 'clusters/{fixed-4', 'crc-failures/bad_map_crc_failure', 'crc-failures/default', 'd-balancer/crush-compat', 'd-balancer/on', 'd-thrash/default/{default', 'fast/fast', 'fast/normal', 'mon_election/classic', 'mon_election/connectivity', 'msgr-failures/fastclose', 'msgr-failures/few', 'msgr-failures/osd-delay', 'msgr-failures/osd-dispatch-delay', 'msgr/async', 'msgr/async-v1only', 'objectstore/bluestore-comp-snappy', 'objectstore/bluestore-comp-zlib', 'objectstore/bluestore-hybrid', 'objectstore/bluestore-stupid', 'openstack}', 'rados', 'rados/thrash-erasure-code-shec/{ceph', 'rados/thrash-erasure-code/{ceph', 'rados/thrash/{0-size-min-size-overrides/2-size-2-min-size', 'rados/thrash/{0-size-min-size-overrides/3-size-2-min-size', 'rados/verify/{centos_latest', 'recovery-overrides/{more-async-partial-recovery}', 'recovery-overrides/{more-async-recovery}', 'recovery-overrides/{more-partial-recovery}', 'supported-random-distro$/{centos_8}', 'supported-random-distro$/{rhel_8}', 'supported-random-distro$/{ubuntu_latest}', 'tasks/mon_recovery', 'thrashers/default', 'thrashers/fastread', 'thrashers/mapgap', 'thrashers/morepggrow', 'thrashers/pggrow', 'thrashosds-health', 'thrashosds-health}', 'validater/valgrind}', 'workloads/admin_socket_objecter_requests}', 'workloads/dedup-io-snaps}', 'workloads/ec-rados-plugin=shec-k=4-m=3-c=2}', 'workloads/ec-radosbench}', 'workloads/ec-small-objects-balanced}', 'workloads/radosbench-high-concurrency}', 'workloads/radosbench}']

Failure: Command failed on smithi097 with status 1: 'sudo yum -y install ceph-radosgw ceph-test ceph ceph-fuse cephfs-java libcephfs_jni1 libcephfs1 librados2 librbd1 python-ceph rbd-fuse sqlite-devel sqlite-devel sqlite-devel sqlite-devel'
1 jobs: ['7211096']
suites: ['clusters/{fixed-2', 'mon_election/connectivity', 'msgr-failures/many', 'msgr/async', 'objectstore/bluestore-comp-zlib', 'openstack}', 'rados', 'rados/basic/{ceph', 'supported-random-distro$/{rhel_8}', 'tasks/readwrite}']

Timeout 3h running clone.client.0/qa/workunits/rados/test.sh
4 jobs: ['7210893', '7210884', '7211089', '7210911']
suites intersection: ['mon_election/connectivity', 'rados']
suites union: ['1-pg-log-overrides/short_pg_log', '2-recovery-overrides/{more-active-recovery}', '3-scrub-overrides/{max-simultaneous-scrubs-2}', 'backoff/peering', 'ceph', 'clusters/9-mons', 'clusters/{fixed-2', 'crc-failures/default', 'd-balancer/on', 'mon_election/connectivity', 'msgr-failures/few', 'msgr-failures/many', 'msgr-failures/osd-dispatch-delay', 'msgr/async-v1only', 'msgr/async-v2only', 'objectstore/bluestore-comp-lz4', 'objectstore/bluestore-comp-zlib', 'objectstore/bluestore-hybrid', 'objectstore/bluestore-stupid', 'openstack}', 'rados', 'rados/basic/{ceph', 'rados/monthrash/{ceph', 'rados/thrash/{0-size-min-size-overrides/3-size-2-min-size', 'supported-random-distro$/{centos_8}', 'supported-random-distro$/{ubuntu_latest}', 'tasks/rados_api_tests}', 'thrashers/default', 'thrashers/sync', 'thrashers/sync-many', 'thrashosds-health', 'workloads/rados_api_tests}']

Failure: Command failed (workunit test erasure-code/test-erasure-code.sh) on smithi136 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=511015c23eef0684a64171b710e432448df2b7c7 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/standalone/erasure-code/test-erasure-code.sh'
1 jobs: ['7210998']
suites: ['rados/standalone/{supported-random-distro$/{rhel_8}', 'workloads/erasure-code}']

Failure: Command failed (workunit test cephadm/test_dashboard_e2e.sh) on smithi098 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=511015c23eef0684a64171b710e432448df2b7c7 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cephadm/test_dashboard_e2e.sh'
2 jobs: ['7210860', '7211008']
suites intersection: ['debug/mgr', 'rados/dashboard/{0-single-container-host', 'tasks/e2e}']
suites union: ['debug/mgr', 'mon_election/classic', 'mon_election/connectivity', 'rados/dashboard/{0-single-container-host', 'random-objectstore$/{bluestore-comp-lz4}', 'random-objectstore$/{bluestore-low-osd-mem-target}', 'tasks/e2e}']

Failure: Command failed (workunit test scrub/osd-mapper.sh) on smithi006 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=511015c23eef0684a64171b710e432448df2b7c7 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/standalone/scrub/osd-mapper.sh'
1 jobs: ['7210899']
suites: ['rados/standalone/{supported-random-distro$/{rhel_8}', 'workloads/scrub}']

Failure: No module named 'tasks'
3 jobs: ['7211098', '7211099', '7211095']
suites intersection: ['rados']
suites union: ['1-pg-log-overrides/normal_pg_log', '2-recovery-overrides/{more-active-recovery}', '3-scrub-overrides/{default}', 'backoff/peering_and_degraded', 'ceph', 'clusters/{fixed-2', 'crc-failures/bad_map_crc_failure', 'd-balancer/crush-compat', 'fast/fast', 'mon_election/classic', 'mon_election/connectivity', 'msgr-failures/osd-delay', 'msgr/async', 'objectstore/bluestore-comp-zstd', 'openstack}', 'rados', 'rados/singleton-nomsgr/{all/cache-fs-trunc', 'rados/thrash-erasure-code-overwrites/{bluestore-bitmap', 'rados/thrash/{0-size-min-size-overrides/2-size-2-min-size', 'recovery-overrides/{more-active-recovery}', 'supported-random-distro$/{centos_8}}', 'supported-random-distro$/{rhel_8}', 'thrashers/default', 'thrashers/mapgap', 'thrashosds-health', 'workloads/cache-snaps-balanced}', 'workloads/ec-small-objects-overwrites}']

Crash: Command failed on smithi008 with status 1: 'rados list-inconsistent-obj 2.6 --format=json'
ceph version 18.0.0-2996-g511015c2 (511015c23eef0684a64171b710e432448df2b7c7) reef (dev)
 1: /lib64/libpthread.so.0(+0x12ce0) [0x7f18a0479ce0]
 2: (MapCacher::MapCacher<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, ceph::buffer::v15_2_0::list>::get_next(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, ceph::buffer::v15_2_0::list>*)+0x101) [0x56068040ea41]
 3: (Scrub::Store::get_errors(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, unsigned long) const+0x2e3) [0x56068040b023]
 4: (Scrub::Store::get_object_errors(long, librados::object_id_t const&, unsigned long) const+0x4cc) [0x56068040bcdc]
 5: (PrimaryLogScrub::get_store_errors(scrub_ls_arg_t const&, scrub_ls_result_t&) const+0x195) [0x5606803f1735]
 6: (PrimaryLogPG::do_scrub_ls(_mosdop::MOSDOp<std::vector<OSDOp, std::allocator<OSDOp> > > const*, OSDOp*)+0x1fc) [0x5606802c759c]
 7: (PrimaryLogPG::do_pg_op(boost::intrusive_ptr<OpRequest>)+0x622) [0x560680300b02]
 8: (PrimaryLogPG::do_op(boost::intrusive_ptr<OpRequest>&)+0xb89) [0x5606803392b9]
 9: (OSD::dequeue_op(boost::intrusive_ptr<PG>, boost::intrusive_ptr<OpRequest>, ThreadPool::TPHandle&)+0x1c6) [0x56068018b236]
 10: (ceph::osd::scheduler::PGOpItem::run(OSD*, OSDShard*, boost::intrusive_ptr<PG>&, ThreadPool::TPHandle&)+0x69) [0x5606804935b9]
 11: (OSD::ShardedOpWQ::_process(unsigned int, ceph::heartbeat_handle_d*)+0x115d) [0x5606801a22dd]
 12: (ShardedThreadPool::shardedthreadpool_worker(unsigned int)+0x43b) [0x56068084aa8b]
 13: (ShardedThreadPool::WorkThreadSharded::entry()+0x14) [0x56068084dcb4]
 14: /lib64/libpthread.so.0(+0x81cf) [0x7f18a046f1cf]
 15: clone()
1 jobs: ['7210869']
suites: ['clusters/{fixed-2', 'mon_election/connectivity', 'msgr-failures/many', 'msgr/async-v2only', 'objectstore/bluestore-comp-lz4', 'openstack}', 'rados', 'rados/basic/{ceph', 'supported-random-distro$/{rhel_8}', 'tasks/scrub_test}']

Failure: failed to complete snap trimming before timeout
1 jobs: ['7210987']
suites: ['ceph', 'clusters/{fixed-2', 'mon_election/classic', 'msgr-failures/few', 'objectstore/bluestore-comp-zstd', 'openstack}', 'rados', 'rados/thrash-erasure-code-isa/{arch/x86_64', 'recovery-overrides/{more-active-recovery}', 'supported-random-distro$/{rhel_8}', 'thrashers/none', 'thrashosds-health', 'workloads/ec-rados-plugin=isa-k=2-m=1}']

Dead: b'2023-03-26T01:31:34.390 DEBUG:teuthology.orchestra.run.smithi154:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 30 ceph --cluster ceph --admin-daemon /var/run/ceph/ceph-osd.3.asok dump_ops_in_flight'
1 jobs: ['7211136']
suites: ['ceph', 'clusters/{fixed-2', 'fast/normal', 'mon_election/connectivity', 'msgr-failures/osd-dispatch-delay', 'openstack}', 'rados', 'rados/thrash-erasure-code-overwrites/{bluestore-bitmap', 'recovery-overrides/{more-partial-recovery}', 'supported-random-distro$/{rhel_8}', 'thrashers/fastread', 'thrashosds-health', 'workloads/ec-snaps-few-objects-overwrites}']

Timeout 3h running clone.client.0/qa/workunits/rbd/test_librbd_python.sh
1 jobs: ['7211082']
suites: ['0-start', '1-tasks', 'mon_election/connectivity', 'rados/upgrade/parallel/{0-random-distro$/{ubuntu_20.04}', 'rados_api', 'rados_loadgenbig', 'rbd_import_export', 'test_rbd_api', 'test_rbd_python}}', 'upgrade-sequence', 'workload/{ec-rados-default']

Valgrind: osd (Leak_PossiblyLost)
1 jobs: ['7211035']
suites: ['2-inject-leak/none', 'centos_latest}', 'rados/valgrind-leaks/{1-start']

Failure: Stale jobs detected, aborting.
1 jobs: ['7211027']
suites: ['0-nvme-loop', 'agent/off', 'fixed-2', 'mon_election/connectivity', 'rados/cephadm/smoke/{0-distro/rhel_8.6_container_tools_3.0', 'start}']

Failure: Command failed (workunit test osd/divergent-priors.sh) on smithi117 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=511015c23eef0684a64171b710e432448df2b7c7 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/standalone/osd/divergent-priors.sh'
1 jobs: ['7210868']
suites: ['rados/standalone/{supported-random-distro$/{ubuntu_latest}', 'workloads/osd}']

Failure: Command failed (workunit test objectstore/test_fuse.sh) on smithi139 with status 139: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=511015c23eef0684a64171b710e432448df2b7c7 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/objectstore/test_fuse.sh'
1 jobs: ['7210977']
suites: ['rados/objectstore/{backends/fusestore', 'supported-random-distro$/{ubuntu_latest}}']

Failure: Command failed (workunit test rados/test_envlibrados_for_rocksdb.sh) on smithi121 with status 2: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=511015c23eef0684a64171b710e432448df2b7c7 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/rados/test_envlibrados_for_rocksdb.sh'
2 jobs: ['7210941', '7210894']
suites intersection: ['mon_election/connectivity', 'msgr-failures/none', 'msgr/async-v2only', 'rados', 'supported-random-distro$/{ubuntu_latest}}', 'test_envlibrados_for_rocksdb}']
suites union: ['mon_election/connectivity', 'msgr-failures/none', 'msgr/async-v2only', 'objectstore/bluestore-bitmap', 'objectstore/bluestore-comp-snappy', 'rados', 'rados/singleton/{all/test_envlibrados_for_rocksdb/{supported/centos_latest', 'rados/singleton/{all/test_envlibrados_for_rocksdb/{supported/rhel_latest', 'supported-random-distro$/{ubuntu_latest}}', 'test_envlibrados_for_rocksdb}']

Failure: Command crashed: 'sudo adjust-ulimits ceph-objectstore-tool --data-path /var/lib/ceph/osd/ceph-2 --journal-path /var/lib/ceph/osd/ceph-2/journal --log-file=/var/log/ceph/objectstore_tool.$$.log --op export-remove --pgid 2.0 --file /home/ubuntu/cephtest/exp.27675.out'
1 jobs: ['7210974']
suites: ['mon_election/classic', 'msgr-failures/many', 'msgr/async-v1only', 'objectstore/bluestore-hybrid', 'rados', 'rados/singleton/{all/divergent_priors2', 'supported-random-distro$/{centos_8}}']